pythia-70m / ndarray-cache.json
Amai
Add weights
cb68c41
Raw History Blame Contribute Delete
29 kB
{
"metadata": {
"ParamSize": 76,
"ParamBytes": 140883968.0,
"BitsPerParam": 16.00348958939165
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 51511296,
"records": [
{
"name": "gpt_neox.embed_in.weight",
"shape": [
50304,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 51511296,
"byteOffset": 0
}
],
"md5sum": "a1619d49d0b0ba376d0f671d12c13e92"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 33103872,
"records": [
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 1024
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 2048
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 3072
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.weight",
"shape": [
1536,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 4096
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
1536
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3072,
"byteOffset": 1576960
},
{
"name": "gpt_neox.layers.0.attention.dense.weight",
"shape": [
512,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 1580032
},
{
"name": "gpt_neox.layers.0.attention.dense.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 2104320
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight",
"shape": [
2048,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 2105344
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 4202496
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight",
"shape": [
512,
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 4206592
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
"shape": [
512
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 6303744
},
{
"name": "gpt_neox.layers.1.input_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 6304768
},
{
"name": "gpt_neox.layers.1.input_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 6305792
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 6306816
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 6307840
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.weight",
"shape": [
1536,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 6308864
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.bias",
"shape": [
1536
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3072,
"byteOffset": 7881728
},
{
"name": "gpt_neox.layers.1.attention.dense.weight",
"shape": [
512,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 7884800
},
{
"name": "gpt_neox.layers.1.attention.dense.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 8409088
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight",
"shape": [
2048,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 8410112
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 10507264
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight",
"shape": [
512,
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 10511360
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
"shape": [
512
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 12608512
},
{
"name": "gpt_neox.layers.2.input_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 12609536
},
{
"name": "gpt_neox.layers.2.input_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 12610560
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 12611584
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 12612608
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.weight",
"shape": [
1536,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 12613632
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.bias",
"shape": [
1536
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3072,
"byteOffset": 14186496
},
{
"name": "gpt_neox.layers.2.attention.dense.weight",
"shape": [
512,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 14189568
},
{
"name": "gpt_neox.layers.2.attention.dense.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 14713856
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight",
"shape": [
2048,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 14714880
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 16812032
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight",
"shape": [
512,
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 16816128
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
"shape": [
512
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 18913280
},
{
"name": "gpt_neox.layers.3.input_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 18914304
},
{
"name": "gpt_neox.layers.3.input_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 18915328
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 18916352
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 18917376
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.weight",
"shape": [
1536,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 18918400
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.bias",
"shape": [
1536
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3072,
"byteOffset": 20491264
},
{
"name": "gpt_neox.layers.3.attention.dense.weight",
"shape": [
512,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 20494336
},
{
"name": "gpt_neox.layers.3.attention.dense.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 21018624
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight",
"shape": [
2048,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 21019648
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 23116800
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight",
"shape": [
512,
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 23120896
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
"shape": [
512
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 25218048
},
{
"name": "gpt_neox.layers.4.input_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 25219072
},
{
"name": "gpt_neox.layers.4.input_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 25220096
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 25221120
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 25222144
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.weight",
"shape": [
1536,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 25223168
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.bias",
"shape": [
1536
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3072,
"byteOffset": 26796032
},
{
"name": "gpt_neox.layers.4.attention.dense.weight",
"shape": [
512,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 26799104
},
{
"name": "gpt_neox.layers.4.attention.dense.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 27323392
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight",
"shape": [
2048,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 27324416
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 29421568
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight",
"shape": [
512,
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 29425664
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
"shape": [
512
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 31522816
},
{
"name": "gpt_neox.layers.5.input_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 31523840
},
{
"name": "gpt_neox.layers.5.input_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 31524864
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 31525888
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 31526912
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.weight",
"shape": [
1536,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1572864,
"byteOffset": 31527936
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.bias",
"shape": [
1536
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3072,
"byteOffset": 33100800
}
],
"md5sum": "ef146e3d692ff0f92b4887051f5ce1aa"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 51511296,
"records": [
{
"name": "embed_out.weight",
"shape": [
50304,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 51511296,
"byteOffset": 0
}
],
"md5sum": "2f016b975538d1e27f57fc84596c2432"
},
{
"dataPath": "params_shard_3.bin",
"format": "raw-shard",
"nbytes": 4726784,
"records": [
{
"name": "gpt_neox.layers.5.attention.dense.weight",
"shape": [
512,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 524288,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.5.attention.dense.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 524288
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight",
"shape": [
2048,
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 525312
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
"shape": [
2048
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 4096,
"byteOffset": 2622464
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight",
"shape": [
512,
2048
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 2097152,
"byteOffset": 2626560
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
"shape": [
512
],
"dtype": "float32",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 4723712
},
{
"name": "gpt_neox.final_layer_norm.weight",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 4724736
},
{
"name": "gpt_neox.final_layer_norm.bias",
"shape": [
512
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1024,
"byteOffset": 4725760
}
],
"md5sum": "4b204eca2568a39d4f462b042c0e9ab7"
}
]
}