Download ndarray-cache.json from AMAI-Lab/pythia-410m-q4f16-1: direct link, hf CLI and curl.
- Browser
- Download file 148 kB
-
https://huggingface.co/AMAI-Lab/pythia-410m-q4f16-1/resolve/main/ndarray-cache.json
- Command line
-
hf download hf://AMAI-Lab/pythia-410m-q4f16-1/ndarray-cache.json
-
curl -L -o ndarray-cache.json https://huggingface.co/AMAI-Lab/pythia-410m-q4f16-1/resolve/main/ndarray-cache.json
148 kB
{
"metadata": {
"ParamSize": 390,
"ParamBytes": 228708352.0,
"BitsPerParam": 4.513973029098056
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 33447936,
"records": [
{
"name": "gpt_neox.embed_in.q_weight",
"shape": [
50304,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25755648,
"byteOffset": 0
},
{
"name": "gpt_neox.embed_in.q_scale",
"shape": [
50304,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3219456,
"byteOffset": 25755648
},
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28975104
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28977152
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28979200
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28981248
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28983296
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 30556160
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 30752768
},
{
"name": "gpt_neox.layers.0.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 30758912
},
{
"name": "gpt_neox.layers.0.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 31283200
},
{
"name": "gpt_neox.layers.0.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 31348736
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 31350784
}
],
"md5sum": "4c3b2ea2f5c2a75292aa897ba7f70428"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 33425408,
"records": [
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 262144
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 270336
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 2367488
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2629632
},
{
"name": "gpt_neox.layers.1.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2631680
},
{
"name": "gpt_neox.layers.1.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2633728
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2635776
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2637824
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 2639872
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 4212736
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 4409344
},
{
"name": "gpt_neox.layers.1.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 4415488
},
{
"name": "gpt_neox.layers.1.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 4939776
},
{
"name": "gpt_neox.layers.1.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 5005312
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 5007360
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 7104512
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 7366656
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 7374848
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 9472000
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9734144
},
{
"name": "gpt_neox.layers.2.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9736192
},
{
"name": "gpt_neox.layers.2.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9738240
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9740288
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9742336
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 9744384
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 11317248
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 11513856
},
{
"name": "gpt_neox.layers.2.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 11520000
},
{
"name": "gpt_neox.layers.2.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 12044288
},
{
"name": "gpt_neox.layers.2.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 12109824
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 12111872
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 14209024
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 14471168
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 14479360
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 16576512
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16838656
},
{
"name": "gpt_neox.layers.3.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16840704
},
{
"name": "gpt_neox.layers.3.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16842752
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16844800
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16846848
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 16848896
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 18421760
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 18618368
},
{
"name": "gpt_neox.layers.3.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 18624512
},
{
"name": "gpt_neox.layers.3.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 19148800
},
{
"name": "gpt_neox.layers.3.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 19214336
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 19216384
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 21313536
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 21575680
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 21583872
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 23681024
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23943168
},
{
"name": "gpt_neox.layers.4.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23945216
},
{
"name": "gpt_neox.layers.4.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23947264
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23949312
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23951360
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 23953408
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 25526272
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 25722880
},
{
"name": "gpt_neox.layers.4.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 25729024
},
{
"name": "gpt_neox.layers.4.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 26253312
},
{
"name": "gpt_neox.layers.4.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26318848
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 26320896
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 28418048
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 28680192
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 28688384
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 30785536
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 31047680
},
{
"name": "gpt_neox.layers.5.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 31049728
},
{
"name": "gpt_neox.layers.5.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 31051776
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 31053824
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 31055872
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 31057920
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 32630784
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 32827392
},
{
"name": "gpt_neox.layers.5.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 32833536
},
{
"name": "gpt_neox.layers.5.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 33357824
},
{
"name": "gpt_neox.layers.5.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 33423360
}
],
"md5sum": "84fc60d24a69d34bf755e43043b00663"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 33155072,
"records": [
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 2097152
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 2359296
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 2367488
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 4464640
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4726784
},
{
"name": "gpt_neox.layers.6.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4728832
},
{
"name": "gpt_neox.layers.6.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4730880
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4732928
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4734976
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 4737024
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 6309888
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 6506496
},
{
"name": "gpt_neox.layers.6.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 6512640
},
{
"name": "gpt_neox.layers.6.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 7036928
},
{
"name": "gpt_neox.layers.6.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 7102464
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 7104512
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 9201664
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 9463808
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 9472000
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 11569152
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11831296
},
{
"name": "gpt_neox.layers.7.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11833344
},
{
"name": "gpt_neox.layers.7.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11835392
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11837440
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11839488
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 11841536
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 13414400
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 13611008
},
{
"name": "gpt_neox.layers.7.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 13617152
},
{
"name": "gpt_neox.layers.7.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 14141440
},
{
"name": "gpt_neox.layers.7.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 14206976
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 14209024
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 16306176
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 16568320
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16576512
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18673664
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18935808
},
{
"name": "gpt_neox.layers.8.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18937856
},
{
"name": "gpt_neox.layers.8.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18939904
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18941952
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18944000
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 18946048
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 20518912
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 20715520
},
{
"name": "gpt_neox.layers.8.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 20721664
},
{
"name": "gpt_neox.layers.8.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 21245952
},
{
"name": "gpt_neox.layers.8.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 21311488
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 21313536
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 23410688
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 23672832
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23681024
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 25778176
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26040320
},
{
"name": "gpt_neox.layers.9.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26042368
},
{
"name": "gpt_neox.layers.9.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26044416
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26046464
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26048512
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 26050560
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 27623424
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 27820032
},
{
"name": "gpt_neox.layers.9.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 27826176
},
{
"name": "gpt_neox.layers.9.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 28350464
},
{
"name": "gpt_neox.layers.9.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28416000
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 28418048
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 30515200
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 30777344
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 30785536
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 32882688
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 33144832
},
{
"name": "gpt_neox.layers.10.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 33146880
},
{
"name": "gpt_neox.layers.10.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 33148928
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 33150976
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 33153024
}
],
"md5sum": "685a0681b04bbc1cbba3e191ca3ba73b"
},
{
"dataPath": "params_shard_3.bin",
"format": "raw-shard",
"nbytes": 33153024,
"records": [
{
"name": "gpt_neox.layers.10.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 1572864
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 1769472
},
{
"name": "gpt_neox.layers.10.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 1775616
},
{
"name": "gpt_neox.layers.10.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 2299904
},
{
"name": "gpt_neox.layers.10.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2365440
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 2367488
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 4464640
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 4726784
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 4734976
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 6832128
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 7094272
},
{
"name": "gpt_neox.layers.11.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 7096320
},
{
"name": "gpt_neox.layers.11.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 7098368
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 7100416
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 7102464
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 7104512
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 8677376
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 8873984
},
{
"name": "gpt_neox.layers.11.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 8880128
},
{
"name": "gpt_neox.layers.11.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 9404416
},
{
"name": "gpt_neox.layers.11.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9469952
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 9472000
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 11569152
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 11831296
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 11839488
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 13936640
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 14198784
},
{
"name": "gpt_neox.layers.12.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 14200832
},
{
"name": "gpt_neox.layers.12.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 14202880
},
{
"name": "gpt_neox.layers.12.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 14204928
},
{
"name": "gpt_neox.layers.12.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 14206976
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 14209024
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 15781888
},
{
"name": "gpt_neox.layers.12.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 15978496
},
{
"name": "gpt_neox.layers.12.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 15984640
},
{
"name": "gpt_neox.layers.12.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 16508928
},
{
"name": "gpt_neox.layers.12.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16574464
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16576512
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18673664
},
{
"name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 18935808
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 18944000
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 21041152
},
{
"name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 21303296
},
{
"name": "gpt_neox.layers.13.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 21305344
},
{
"name": "gpt_neox.layers.13.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 21307392
},
{
"name": "gpt_neox.layers.13.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 21309440
},
{
"name": "gpt_neox.layers.13.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 21311488
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 21313536
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 22886400
},
{
"name": "gpt_neox.layers.13.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 23083008
},
{
"name": "gpt_neox.layers.13.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 23089152
},
{
"name": "gpt_neox.layers.13.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 23613440
},
{
"name": "gpt_neox.layers.13.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23678976
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23681024
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 25778176
},
{
"name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 26040320
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 26048512
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 28145664
},
{
"name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28407808
},
{
"name": "gpt_neox.layers.14.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28409856
},
{
"name": "gpt_neox.layers.14.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28411904
},
{
"name": "gpt_neox.layers.14.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28413952
},
{
"name": "gpt_neox.layers.14.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28416000
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28418048
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 29990912
},
{
"name": "gpt_neox.layers.14.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 30187520
},
{
"name": "gpt_neox.layers.14.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 30193664
},
{
"name": "gpt_neox.layers.14.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 30717952
},
{
"name": "gpt_neox.layers.14.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 30783488
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 30785536
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 32882688
},
{
"name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 33144832
}
],
"md5sum": "a5fd0090ee7c8de95c415675151b3473"
},
{
"dataPath": "params_shard_4.bin",
"format": "raw-shard",
"nbytes": 33155072,
"records": [
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 2097152
},
{
"name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2359296
},
{
"name": "gpt_neox.layers.15.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2361344
},
{
"name": "gpt_neox.layers.15.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2363392
},
{
"name": "gpt_neox.layers.15.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2365440
},
{
"name": "gpt_neox.layers.15.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 2367488
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 2369536
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 3942400
},
{
"name": "gpt_neox.layers.15.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 4139008
},
{
"name": "gpt_neox.layers.15.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 4145152
},
{
"name": "gpt_neox.layers.15.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 4669440
},
{
"name": "gpt_neox.layers.15.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4734976
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 4737024
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 6834176
},
{
"name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 7096320
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 7104512
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 9201664
},
{
"name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9463808
},
{
"name": "gpt_neox.layers.16.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9465856
},
{
"name": "gpt_neox.layers.16.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9467904
},
{
"name": "gpt_neox.layers.16.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9469952
},
{
"name": "gpt_neox.layers.16.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 9472000
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 9474048
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 11046912
},
{
"name": "gpt_neox.layers.16.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 11243520
},
{
"name": "gpt_neox.layers.16.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 11249664
},
{
"name": "gpt_neox.layers.16.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 11773952
},
{
"name": "gpt_neox.layers.16.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11839488
},
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 11841536
},
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 13938688
},
{
"name": "gpt_neox.layers.16.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 14200832
},
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 14209024
},
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 16306176
},
{
"name": "gpt_neox.layers.16.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16568320
},
{
"name": "gpt_neox.layers.17.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16570368
},
{
"name": "gpt_neox.layers.17.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16572416
},
{
"name": "gpt_neox.layers.17.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16574464
},
{
"name": "gpt_neox.layers.17.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 16576512
},
{
"name": "gpt_neox.layers.17.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 16578560
},
{
"name": "gpt_neox.layers.17.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 18151424
},
{
"name": "gpt_neox.layers.17.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 18348032
},
{
"name": "gpt_neox.layers.17.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 18354176
},
{
"name": "gpt_neox.layers.17.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 18878464
},
{
"name": "gpt_neox.layers.17.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18944000
},
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 18946048
},
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 21043200
},
{
"name": "gpt_neox.layers.17.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 21305344
},
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 21313536
},
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 23410688
},
{
"name": "gpt_neox.layers.17.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23672832
},
{
"name": "gpt_neox.layers.18.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23674880
},
{
"name": "gpt_neox.layers.18.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23676928
},
{
"name": "gpt_neox.layers.18.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23678976
},
{
"name": "gpt_neox.layers.18.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 23681024
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 23683072
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 25255936
},
{
"name": "gpt_neox.layers.18.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 25452544
},
{
"name": "gpt_neox.layers.18.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 25458688
},
{
"name": "gpt_neox.layers.18.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 25982976
},
{
"name": "gpt_neox.layers.18.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26048512
},
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 26050560
},
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 28147712
},
{
"name": "gpt_neox.layers.18.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 28409856
},
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 28418048
},
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 30515200
},
{
"name": "gpt_neox.layers.18.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 30777344
},
{
"name": "gpt_neox.layers.19.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 30779392
},
{
"name": "gpt_neox.layers.19.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 30781440
},
{
"name": "gpt_neox.layers.19.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 30783488
},
{
"name": "gpt_neox.layers.19.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 30785536
},
{
"name": "gpt_neox.layers.19.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 30787584
},
{
"name": "gpt_neox.layers.19.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 32360448
},
{
"name": "gpt_neox.layers.19.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 32557056
},
{
"name": "gpt_neox.layers.19.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 32563200
},
{
"name": "gpt_neox.layers.19.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 33087488
},
{
"name": "gpt_neox.layers.19.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 33153024
}
],
"md5sum": "9138a0950e3a35eb9051beedf39ba30a"
},
{
"dataPath": "params_shard_5.bin",
"format": "raw-shard",
"nbytes": 25755648,
"records": [
{
"name": "embed_out.q_weight",
"shape": [
50304,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25755648,
"byteOffset": 0
}
],
"md5sum": "69270e949e1cc6f7e240cd1c9377d5fa"
},
{
"dataPath": "params_shard_6.bin",
"format": "raw-shard",
"nbytes": 33150976,
"records": [
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 2097152
},
{
"name": "gpt_neox.layers.19.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 2359296
},
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 2367488
},
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 4464640
},
{
"name": "gpt_neox.layers.19.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4726784
},
{
"name": "gpt_neox.layers.20.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4728832
},
{
"name": "gpt_neox.layers.20.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4730880
},
{
"name": "gpt_neox.layers.20.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4732928
},
{
"name": "gpt_neox.layers.20.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 4734976
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 4737024
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 6309888
},
{
"name": "gpt_neox.layers.20.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 6506496
},
{
"name": "gpt_neox.layers.20.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 6512640
},
{
"name": "gpt_neox.layers.20.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 7036928
},
{
"name": "gpt_neox.layers.20.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 7102464
},
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 7104512
},
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 9201664
},
{
"name": "gpt_neox.layers.20.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 9463808
},
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 9472000
},
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 11569152
},
{
"name": "gpt_neox.layers.20.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11831296
},
{
"name": "gpt_neox.layers.21.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11833344
},
{
"name": "gpt_neox.layers.21.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11835392
},
{
"name": "gpt_neox.layers.21.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11837440
},
{
"name": "gpt_neox.layers.21.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 11839488
},
{
"name": "gpt_neox.layers.21.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 11841536
},
{
"name": "gpt_neox.layers.21.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 13414400
},
{
"name": "gpt_neox.layers.21.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 13611008
},
{
"name": "gpt_neox.layers.21.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 13617152
},
{
"name": "gpt_neox.layers.21.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 14141440
},
{
"name": "gpt_neox.layers.21.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 14206976
},
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 14209024
},
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 16306176
},
{
"name": "gpt_neox.layers.21.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 16568320
},
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16576512
},
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 18673664
},
{
"name": "gpt_neox.layers.21.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18935808
},
{
"name": "gpt_neox.layers.22.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18937856
},
{
"name": "gpt_neox.layers.22.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18939904
},
{
"name": "gpt_neox.layers.22.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18941952
},
{
"name": "gpt_neox.layers.22.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 18944000
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 18946048
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 20518912
},
{
"name": "gpt_neox.layers.22.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 20715520
},
{
"name": "gpt_neox.layers.22.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 20721664
},
{
"name": "gpt_neox.layers.22.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 21245952
},
{
"name": "gpt_neox.layers.22.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 21311488
},
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 21313536
},
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 23410688
},
{
"name": "gpt_neox.layers.22.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 23672832
},
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23681024
},
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 25778176
},
{
"name": "gpt_neox.layers.22.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26040320
},
{
"name": "gpt_neox.layers.23.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26042368
},
{
"name": "gpt_neox.layers.23.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26044416
},
{
"name": "gpt_neox.layers.23.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26046464
},
{
"name": "gpt_neox.layers.23.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 26048512
},
{
"name": "gpt_neox.layers.23.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 26050560
},
{
"name": "gpt_neox.layers.23.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 27623424
},
{
"name": "gpt_neox.layers.23.attention.query_key_value.bias",
"shape": [
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 6144,
"byteOffset": 27820032
},
{
"name": "gpt_neox.layers.23.attention.dense.q_weight",
"shape": [
1024,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 27826176
},
{
"name": "gpt_neox.layers.23.attention.dense.q_scale",
"shape": [
1024,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 65536,
"byteOffset": 28350464
},
{
"name": "gpt_neox.layers.23.attention.dense.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28416000
},
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_weight",
"shape": [
4096,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 28418048
},
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.q_scale",
"shape": [
4096,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 30515200
},
{
"name": "gpt_neox.layers.23.mlp.dense_h_to_4h.bias",
"shape": [
4096
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 8192,
"byteOffset": 30777344
},
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_weight",
"shape": [
1024,
512
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 30785536
},
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.q_scale",
"shape": [
1024,
128
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 262144,
"byteOffset": 32882688
},
{
"name": "gpt_neox.layers.23.mlp.dense_4h_to_h.bias",
"shape": [
1024
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 33144832
},
{
"name": "gpt_neox.final_layer_norm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 33146880
},
{
"name": "gpt_neox.final_layer_norm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 33148928
}
],
"md5sum": "a81a6db3611f269bc5690557a6fa5877"
},
{
"dataPath": "params_shard_7.bin",
"format": "raw-shard",
"nbytes": 3219456,
"records": [
{
"name": "embed_out.q_scale",
"shape": [
50304,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3219456,
"byteOffset": 0
}
],
"md5sum": "ecc56611193223a982231f3f2ec7b649"
}
]
}
{
"metadata": {
"ParamSize": 390,
"ParamBytes": 228708352.0,
"BitsPerParam": 4.513973029098056
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 33447936,
"records": [
{
"name": "gpt_neox.embed_in.q_weight",
"shape": [
50304,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 25755648,
"byteOffset": 0
},
{
"name": "gpt_neox.embed_in.q_scale",
"shape": [
50304,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3219456,
"byteOffset": 25755648
},
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28975104
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28977152
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28979200
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
1024
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2048,
"byteOffset": 28981248
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_weight",
"shape": [
3072,
128
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 28983296
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_scale",
"shape": [
3072,
32
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 196608,
"byteOffset": 30556160
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
3072