Download ndarray-cache-b16.json from AMAI-Lab/pythia-160m: direct link, hf CLI and curl.
- Browser
- Download file 56.3 kB
-
https://huggingface.co/AMAI-Lab/pythia-160m/resolve/main/ndarray-cache-b16.json
- Command line
-
hf download hf://AMAI-Lab/pythia-160m/ndarray-cache-b16.json
-
curl -L -o ndarray-cache-b16.json https://huggingface.co/AMAI-Lab/pythia-160m/resolve/main/ndarray-cache-b16.json
56.3 kB
{
"metadata": {
"ParamSize": 148,
"ParamBytes": 324738048.0,
"BitsPerParam": 16.00454205660538
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 77266944,
"records": [
{
"name": "gpt_neox.embed_in.weight",
"shape": [
50304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 77266944,
"byteOffset": 0
}
],
"md5sum": "2481121002c12eed2951b729fa17b644"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 33082368,
"records": [
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 1536
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 3072
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4608
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 6144
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 3545088
},
{
"name": "gpt_neox.layers.0.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 3549696
},
{
"name": "gpt_neox.layers.0.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4729344
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 4730880
},
{
"name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 9455616
},
{
"name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 14174208
},
{
"name": "gpt_neox.layers.1.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14175744
},
{
"name": "gpt_neox.layers.1.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14177280
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14178816
},
{
"name": "gpt_neox.layers.1.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14180352
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 14181888
},
{
"name": "gpt_neox.layers.1.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 17720832
},
{
"name": "gpt_neox.layers.1.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 17725440
},
{
"name": "gpt_neox.layers.1.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 18905088
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 18906624
},
{
"name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 23625216
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 23631360
},
{
"name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 28349952
},
{
"name": "gpt_neox.layers.2.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 28351488
},
{
"name": "gpt_neox.layers.2.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 28353024
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 28354560
},
{
"name": "gpt_neox.layers.2.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 28356096
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 28357632
},
{
"name": "gpt_neox.layers.2.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 31896576
},
{
"name": "gpt_neox.layers.2.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 31901184
},
{
"name": "gpt_neox.layers.2.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 33080832
}
],
"md5sum": "f152a892aaec5607fd2b89419435780a"
},
{
"dataPath": "params_shard_2.bin",
"format": "raw-shard",
"nbytes": 33076224,
"records": [
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 4718592
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 4724736
},
{
"name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 9443328
},
{
"name": "gpt_neox.layers.3.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9444864
},
{
"name": "gpt_neox.layers.3.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9446400
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9447936
},
{
"name": "gpt_neox.layers.3.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 9451008
},
{
"name": "gpt_neox.layers.3.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 12989952
},
{
"name": "gpt_neox.layers.3.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 12994560
},
{
"name": "gpt_neox.layers.3.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14174208
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 14175744
},
{
"name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 18894336
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 18900480
},
{
"name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 23619072
},
{
"name": "gpt_neox.layers.4.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 23620608
},
{
"name": "gpt_neox.layers.4.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 23622144
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 23623680
},
{
"name": "gpt_neox.layers.4.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 23625216
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 23626752
},
{
"name": "gpt_neox.layers.4.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 27165696
},
{
"name": "gpt_neox.layers.4.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 27170304
},
{
"name": "gpt_neox.layers.4.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 28349952
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 28351488
},
{
"name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 33070080
}
],
"md5sum": "57ece0af7da914417b0f9080c0bdac26"
},
{
"dataPath": "params_shard_3.bin",
"format": "raw-shard",
"nbytes": 33077760,
"records": [
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 4718592
},
{
"name": "gpt_neox.layers.5.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4720128
},
{
"name": "gpt_neox.layers.5.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4721664
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4723200
},
{
"name": "gpt_neox.layers.5.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4724736
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 4726272
},
{
"name": "gpt_neox.layers.5.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 8265216
},
{
"name": "gpt_neox.layers.5.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 8269824
},
{
"name": "gpt_neox.layers.5.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 9451008
},
{
"name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 14169600
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 14175744
},
{
"name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 18894336
},
{
"name": "gpt_neox.layers.6.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 18895872
},
{
"name": "gpt_neox.layers.6.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 18897408
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 18898944
},
{
"name": "gpt_neox.layers.6.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 18900480
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 18902016
},
{
"name": "gpt_neox.layers.6.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 22440960
},
{
"name": "gpt_neox.layers.6.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 22445568
},
{
"name": "gpt_neox.layers.6.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 23625216
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 23626752
},
{
"name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 28345344
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 28351488
},
{
"name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 33070080
},
{
"name": "gpt_neox.layers.7.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 33071616
},
{
"name": "gpt_neox.layers.7.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 33073152
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 33074688
},
{
"name": "gpt_neox.layers.7.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 33076224
}
],
"md5sum": "773204e0e5615a84b4109b95d21abb1e"
},
{
"dataPath": "params_shard_4.bin",
"format": "raw-shard",
"nbytes": 33076224,
"records": [
{
"name": "gpt_neox.layers.7.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.7.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 3538944
},
{
"name": "gpt_neox.layers.7.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 3543552
},
{
"name": "gpt_neox.layers.7.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4723200
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 4724736
},
{
"name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 9443328
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 14168064
},
{
"name": "gpt_neox.layers.8.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14169600
},
{
"name": "gpt_neox.layers.8.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14171136
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14172672
},
{
"name": "gpt_neox.layers.8.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14174208
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 14175744
},
{
"name": "gpt_neox.layers.8.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 17714688
},
{
"name": "gpt_neox.layers.8.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 17719296
},
{
"name": "gpt_neox.layers.8.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 18898944
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 18900480
},
{
"name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 23619072
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 23625216
},
{
"name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 28343808
},
{
"name": "gpt_neox.layers.9.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 28345344
},
{
"name": "gpt_neox.layers.9.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 28346880
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 28348416
},
{
"name": "gpt_neox.layers.9.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 28349952
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 28351488
},
{
"name": "gpt_neox.layers.9.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 31890432
},
{
"name": "gpt_neox.layers.9.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 31895040
},
{
"name": "gpt_neox.layers.9.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 33074688
}
],
"md5sum": "9be3a171e75708baaaf6249630cc7a0d"
},
{
"dataPath": "params_shard_5.bin",
"format": "raw-shard",
"nbytes": 33076224,
"records": [
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 4718592
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 4724736
},
{
"name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 9443328
},
{
"name": "gpt_neox.layers.10.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9444864
},
{
"name": "gpt_neox.layers.10.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9446400
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9447936
},
{
"name": "gpt_neox.layers.10.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 9449472
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 9451008
},
{
"name": "gpt_neox.layers.10.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 12989952
},
{
"name": "gpt_neox.layers.10.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 12994560
},
{
"name": "gpt_neox.layers.10.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 14174208
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 14175744
},
{
"name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 18894336
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 18900480
},
{
"name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 23619072
},
{
"name": "gpt_neox.layers.11.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 23620608
},
{
"name": "gpt_neox.layers.11.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 23622144
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 23623680
},
{
"name": "gpt_neox.layers.11.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 23625216
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 23626752
},
{
"name": "gpt_neox.layers.11.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 27165696
},
{
"name": "gpt_neox.layers.11.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1179648,
"byteOffset": 27170304
},
{
"name": "gpt_neox.layers.11.attention.dense.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 28349952
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.weight",
"shape": [
3072,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 28351488
},
{
"name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
"shape": [
3072
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 6144,
"byteOffset": 33070080
}
],
"md5sum": "d3807c10c1e4cf1ed703d786dad97dd5"
},
{
"dataPath": "params_shard_6.bin",
"format": "raw-shard",
"nbytes": 77266944,
"records": [
{
"name": "embed_out.weight",
"shape": [
50304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 77266944,
"byteOffset": 0
}
],
"md5sum": "084dbad7777340578a2501ea88f226dd"
},
{
"dataPath": "params_shard_7.bin",
"format": "raw-shard",
"nbytes": 4723200,
"records": [
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.weight",
"shape": [
768,
3072
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4718592,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
"shape": [
768
],
"dtype": "bfloat16",
"format": "raw",
"nbytes": 1536,
"byteOffset": 4718592
},
{
"name": "gpt_neox.final_layer_norm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4720128
},
{
"name": "gpt_neox.final_layer_norm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4721664
}
],
"md5sum": "5290d766289dc0d43e7407c8dc728b2e"
}
]
}
{
"metadata": {
"ParamSize": 148,
"ParamBytes": 324738048.0,
"BitsPerParam": 16.00454205660538
},
"records": [
{
"dataPath": "params_shard_0.bin",
"format": "raw-shard",
"nbytes": 77266944,
"records": [
{
"name": "gpt_neox.embed_in.weight",
"shape": [
50304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 77266944,
"byteOffset": 0
}
],
"md5sum": "2481121002c12eed2951b729fa17b644"
},
{
"dataPath": "params_shard_1.bin",
"format": "raw-shard",
"nbytes": 33082368,
"records": [
{
"name": "gpt_neox.layers.0.input_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 0
},
{
"name": "gpt_neox.layers.0.input_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 1536
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.weight",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 3072
},
{
"name": "gpt_neox.layers.0.post_attention_layernorm.bias",
"shape": [
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 1536,
"byteOffset": 4608
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.weight",
"shape": [
2304,
768
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 3538944,
"byteOffset": 6144
},
{
"name": "gpt_neox.layers.0.attention.query_key_value.bias",
"shape": [
2304
],
"dtype": "float16",
"format": "f32-to-bf16",
"nbytes": 4608,
"byteOffset": 3545088
},
{
"name": "gpt_neox.layers.0.attention.dense.weight",
"shape": [
768,
768
],
"dtype": "float16",
"format": "f32-to-bf16",