Download ndarray-cache-b16.json from AMAI-Lab/pythia-160m-q4f16-1: direct link, hf CLI and curl.
- Browser
- Download file 74.6 kB
-
https://huggingface.co/AMAI-Lab/pythia-160m-q4f16-1/resolve/main/ndarray-cache-b16.json
- Command line
-
hf download hf://AMAI-Lab/pythia-160m-q4f16-1/ndarray-cache-b16.json
-
curl -L -o ndarray-cache-b16.json https://huggingface.co/AMAI-Lab/pythia-160m-q4f16-1/resolve/main/ndarray-cache-b16.json
74.6 kB
{
"metadata": {
"ParamSize": 198,
"ParamBytes": 91573248.0,
"BitsPerParam": 4.513138844992856
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 32406528,
"records": [
{
"name": "gpt_neox.embed_in.q_weight",
"shape": [
50304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 19316736,
"byteOffset": 0
},
{
"name": "gpt_neox.embed_in.q_scale",
"shape": [
50304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2414592,
"byteOffset": 19316736
},
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21731328
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21732864
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21734400
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21735936
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 21737472
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 22622208
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 22732800
},
{
"name": "gpt_neox.layers.0.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 22737408
},
{
"name": "gpt_neox.layers.0.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 23032320
},
{
"name": "gpt_neox.layers.0.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 23069184
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 23070720
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 24250368
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 24397824
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 24403968
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 25583616
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 25731072
},
{
"name": "gpt_neox.layers.1.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 25732608
},
{
"name": "gpt_neox.layers.1.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 25734144
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 25735680
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 25737216
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 25738752
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 26623488
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 26734080
},
{
"name": "gpt_neox.layers.1.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 26738688
},
{
"name": "gpt_neox.layers.1.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 27033600
},
{
"name": "gpt_neox.layers.1.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 27070464
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 27072000
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 28251648
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 28399104
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 28405248
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 29584896
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 29732352
},
{
"name": "gpt_neox.layers.2.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 29733888
},
{
"name": "gpt_neox.layers.2.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 29735424
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 29736960
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 29738496
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 29740032
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 30624768
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 30735360
},
{
"name": "gpt_neox.layers.2.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 30739968
},
{
"name": "gpt_neox.layers.2.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 31034880
},
{
"name": "gpt_neox.layers.2.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 31071744
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 31073280
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 32252928
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 32400384
}
],
"md5sum": "f0227163eab2db5bdea0284280f0f447"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 33345024,
"records": [
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 1179648
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 1327104
},
{
"name": "gpt_neox.layers.3.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 1328640
},
{
"name": "gpt_neox.layers.3.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 1330176
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 1331712
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 1333248
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 1334784
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 2219520
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 2330112
},
{
"name": "gpt_neox.layers.3.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 2334720
},
{
"name": "gpt_neox.layers.3.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 2629632
},
{
"name": "gpt_neox.layers.3.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 2666496
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 2668032
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 3847680
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 3995136
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 4001280
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 5180928
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 5328384
},
{
"name": "gpt_neox.layers.4.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 5329920
},
{
"name": "gpt_neox.layers.4.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 5331456
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 5332992
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 5334528
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 5336064
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 6220800
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 6331392
},
{
"name": "gpt_neox.layers.4.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 6336000
},
{
"name": "gpt_neox.layers.4.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 6630912
},
{
"name": "gpt_neox.layers.4.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 6667776
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 6669312
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 7848960
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 7996416
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 8002560
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 9182208
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 9329664
},
{
"name": "gpt_neox.layers.5.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9331200
},
{
"name": "gpt_neox.layers.5.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9332736
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9334272
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9335808
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 9337344
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 10222080
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 10332672
},
{
"name": "gpt_neox.layers.5.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 10337280
},
{
"name": "gpt_neox.layers.5.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 10632192
},
{
"name": "gpt_neox.layers.5.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 10669056
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 10670592
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 11850240
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 11997696
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 12003840
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 13183488
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 13330944
},
{
"name": "gpt_neox.layers.6.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 13332480
},
{
"name": "gpt_neox.layers.6.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 13334016
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 13335552
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 13337088
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 13338624
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 14223360
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 14333952
},
{
"name": "gpt_neox.layers.6.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 14338560
},
{
"name": "gpt_neox.layers.6.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 14633472
},
{
"name": "gpt_neox.layers.6.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14670336
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 14671872
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 15851520
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 15998976
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 16005120
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 17184768
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 17332224
},
{
"name": "gpt_neox.layers.7.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 17333760
},
{
"name": "gpt_neox.layers.7.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 17335296
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 17336832
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 17338368
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 17339904
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 18224640
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 18335232
},
{
"name": "gpt_neox.layers.7.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 18339840
},
{
"name": "gpt_neox.layers.7.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 18634752
},
{
"name": "gpt_neox.layers.7.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 18671616
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 18673152
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 19852800
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 20000256
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 20006400
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 21186048
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 21333504
},
{
"name": "gpt_neox.layers.8.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21335040
},
{
"name": "gpt_neox.layers.8.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21336576
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21338112
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21339648
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 21341184
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 22225920
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 22336512
},
{
"name": "gpt_neox.layers.8.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 22341120
},
{
"name": "gpt_neox.layers.8.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 22636032
},
{
"name": "gpt_neox.layers.8.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 22672896
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 22674432
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 23854080
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 24001536
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 24007680
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 25187328
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 25334784
},
{
"name": "gpt_neox.layers.9.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 25336320
},
{
"name": "gpt_neox.layers.9.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 25337856
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 25339392
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 25340928
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 25342464
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 26227200
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 26337792
},
{
"name": "gpt_neox.layers.9.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 26342400
},
{
"name": "gpt_neox.layers.9.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 26637312
},
{
"name": "gpt_neox.layers.9.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 26674176
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 26675712
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 27855360
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 28002816
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 28008960
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 29188608
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 29336064
},
{
"name": "gpt_neox.layers.10.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 29337600
},
{
"name": "gpt_neox.layers.10.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 29339136
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 29340672
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 29342208
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 29343744
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 30228480
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 30339072
},
{
"name": "gpt_neox.layers.10.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 30343680
},
{
"name": "gpt_neox.layers.10.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 30638592
},
{
"name": "gpt_neox.layers.10.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 30675456
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 30676992
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 31856640
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 32004096
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 32010240
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 33189888
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 33337344
},
{
"name": "gpt_neox.layers.11.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 33338880
},
{
"name": "gpt_neox.layers.11.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 33340416
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 33341952
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 33343488
}
],
"md5sum": "941fc9dffde95be24bd0b5d3febbbe15"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 25729536,
"records": [
{
"name": "gpt_neox.layers.11.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 884736
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 995328
},
{
"name": "gpt_neox.layers.11.attention.dense.q_weight",
"shape": [
768,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 294912,
"byteOffset": 999936
},
{
"name": "gpt_neox.layers.11.attention.dense.q_scale",
"shape": [
768,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 36864,
"byteOffset": 1294848
},
{
"name": "gpt_neox.layers.11.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 1331712
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_weight",
"shape": [
3072,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 1333248
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.q_scale",
"shape": [
3072,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 2512896
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 2660352
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_weight",
"shape": [
768,
384
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 2666496
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.q_scale",
"shape": [
768,
96
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 147456,
"byteOffset": 3846144
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 3993600
},
{
"name": "gpt_neox.final_layer_norm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 3995136
},
{
"name": "gpt_neox.final_layer_norm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 3996672
},
{
"name": "embed_out.q_weight",
"shape": [
50304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 19316736,
"byteOffset": 3998208
},
{
"name": "embed_out.q_scale",
"shape": [
50304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2414592,
"byteOffset": 23314944
}
],
"md5sum": "8cf5b4f860269d72cf81ee713307e594"
}
]
}
{
"metadata": {
"ParamSize": 198,
"ParamBytes": 91573248.0,
"BitsPerParam": 4.513138844992856
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 32406528,
"records": [
{
"name": "gpt_neox.embed_in.q_weight",
"shape": [
50304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 19316736,
"byteOffset": 0
},
{
"name": "gpt_neox.embed_in.q_scale",
"shape": [
50304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2414592,
"byteOffset": 19316736
},
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21731328
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21732864
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21734400
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 21735936
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_weight",
"shape": [
2304,
96
],
"dtype": "uint32",
"format": "f32-to-bf16",
"nbytes": 884736,
"byteOffset": 21737472
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.q_scale",
"shape": [
2304,
24
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 110592,
"byteOffset": 22622208
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
2304