Amai commited on
Commit
8121406
·
1 Parent(s): 2e70cf4

Add weights

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
EleutherAI_pythia-1b-android-q0f16.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42d3205accf2d755c276bbc489a63f713b69e283b49bedaa79c719767113afc8
3
+ size 321052
ir_dump/debug-final.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase0.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase1.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase2.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase3.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase4.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase5.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/kernel.cl ADDED
The diff for this file is too large to render. See raw diff
 
mlc-chat-config.json ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "0.1.0",
3
+ "model_type": "gpt_neox",
4
+ "quantization": "q0f16",
5
+ "model_config": {
6
+ "use_parallel_residual": true,
7
+ "hidden_size": 2048,
8
+ "intermediate_size": 8192,
9
+ "num_attention_heads": 8,
10
+ "num_hidden_layers": 16,
11
+ "layer_norm_eps": 1e-05,
12
+ "vocab_size": 50304,
13
+ "rotary_pct": 0.25,
14
+ "position_embedding_base": 10000,
15
+ "context_window_size": 2048,
16
+ "head_dim": 256,
17
+ "prefill_chunk_size": 2048,
18
+ "tensor_parallel_shards": 1,
19
+ "ffn_out_dtype": "float32",
20
+ "max_batch_size": 128
21
+ },
22
+ "vocab_size": 50304,
23
+ "context_window_size": 2048,
24
+ "sliding_window_size": -1,
25
+ "prefill_chunk_size": 2048,
26
+ "attention_sink_size": -1,
27
+ "tensor_parallel_shards": 1,
28
+ "pipeline_parallel_stages": 1,
29
+ "temperature": 1.0,
30
+ "presence_penalty": 0.0,
31
+ "frequency_penalty": 0.0,
32
+ "repetition_penalty": 1.0,
33
+ "top_p": 1.0,
34
+ "tokenizer_files": [
35
+ "tokenizer.json",
36
+ "tokenizer_config.json"
37
+ ],
38
+ "tokenizer_info": {
39
+ "token_postproc_method": "byte_level",
40
+ "prepend_space_in_encode": false,
41
+ "strip_space_in_decode": false
42
+ },
43
+ "conv_template": {
44
+ "name": "gpt2",
45
+ "system_template": "{system_message}",
46
+ "system_message": "",
47
+ "system_prefix_token_ids": null,
48
+ "add_role_after_system_message": true,
49
+ "roles": {
50
+ "user": "",
51
+ "assistant": ""
52
+ },
53
+ "role_templates": {
54
+ "user": "{user_message}",
55
+ "assistant": "{assistant_message}",
56
+ "tool": "{tool_message}"
57
+ },
58
+ "messages": [],
59
+ "seps": [
60
+ ""
61
+ ],
62
+ "role_content_sep": "",
63
+ "role_empty_sep": "",
64
+ "stop_str": [
65
+ "</s>"
66
+ ],
67
+ "stop_token_ids": [
68
+ 50256
69
+ ],
70
+ "function_string": "",
71
+ "use_function_calling": false
72
+ },
73
+ "pad_token_id": 0,
74
+ "bos_token_id": 0,
75
+ "eos_token_id": 0
76
+ }
ndarray-cache-b16.json ADDED
@@ -0,0 +1,2483 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "ParamSize": 196,
4
+ "ParamBytes": 2023890944.0,
5
+ "BitsPerParam": 16.002590914795338
6
+ },
7
+ "records": [
8
+ {
9
+ "dataPath": "params_shard_0.bin",
10
+ "format": "raw-shard",
11
+ "nbytes": 206045184,
12
+ "records": [
13
+ {
14
+ "name": "gpt_neox.embed_in.weight",
15
+ "shape": [
16
+ 50304,
17
+ 2048
18
+ ],
19
+ "dtype": "float16",
20
+ "format": "f32-to-bf16",
21
+ "nbytes": 206045184,
22
+ "byteOffset": 0
23
+ }
24
+ ],
25
+ "md5sum": "da22c1a38504f2cdb102f78567ef6329"
26
+ },
27
+ {
28
+ "dataPath": "params_shard_1.bin",
29
+ "format": "raw-shard",
30
+ "nbytes": 25194496,
31
+ "records": [
32
+ {
33
+ "name": "gpt_neox.layers.0.input_layernorm.weight",
34
+ "shape": [
35
+ 2048
36
+ ],
37
+ "dtype": "float16",
38
+ "format": "f32-to-bf16",
39
+ "nbytes": 4096,
40
+ "byteOffset": 0
41
+ },
42
+ {
43
+ "name": "gpt_neox.layers.0.input_layernorm.bias",
44
+ "shape": [
45
+ 2048
46
+ ],
47
+ "dtype": "float16",
48
+ "format": "f32-to-bf16",
49
+ "nbytes": 4096,
50
+ "byteOffset": 4096
51
+ },
52
+ {
53
+ "name": "gpt_neox.layers.0.post_attention_layernorm.weight",
54
+ "shape": [
55
+ 2048
56
+ ],
57
+ "dtype": "float16",
58
+ "format": "f32-to-bf16",
59
+ "nbytes": 4096,
60
+ "byteOffset": 8192
61
+ },
62
+ {
63
+ "name": "gpt_neox.layers.0.post_attention_layernorm.bias",
64
+ "shape": [
65
+ 2048
66
+ ],
67
+ "dtype": "float16",
68
+ "format": "f32-to-bf16",
69
+ "nbytes": 4096,
70
+ "byteOffset": 12288
71
+ },
72
+ {
73
+ "name": "gpt_neox.layers.0.attention.query_key_value.weight",
74
+ "shape": [
75
+ 6144,
76
+ 2048
77
+ ],
78
+ "dtype": "float16",
79
+ "format": "f32-to-bf16",
80
+ "nbytes": 25165824,
81
+ "byteOffset": 16384
82
+ },
83
+ {
84
+ "name": "gpt_neox.layers.0.attention.query_key_value.bias",
85
+ "shape": [
86
+ 6144
87
+ ],
88
+ "dtype": "float16",
89
+ "format": "f32-to-bf16",
90
+ "nbytes": 12288,
91
+ "byteOffset": 25182208
92
+ }
93
+ ],
94
+ "md5sum": "e2d7cc2fa350f011ed0dbee17a7ac7e8"
95
+ },
96
+ {
97
+ "dataPath": "params_shard_2.bin",
98
+ "format": "raw-shard",
99
+ "nbytes": 33554432,
100
+ "records": [
101
+ {
102
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight",
103
+ "shape": [
104
+ 8192,
105
+ 2048
106
+ ],
107
+ "dtype": "float16",
108
+ "format": "f32-to-bf16",
109
+ "nbytes": 33554432,
110
+ "byteOffset": 0
111
+ }
112
+ ],
113
+ "md5sum": "e4a3272a25b09a7f71da4953d754e374"
114
+ },
115
+ {
116
+ "dataPath": "params_shard_3.bin",
117
+ "format": "raw-shard",
118
+ "nbytes": 33554432,
119
+ "records": [
120
+ {
121
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight",
122
+ "shape": [
123
+ 2048,
124
+ 8192
125
+ ],
126
+ "dtype": "float16",
127
+ "format": "f32-to-bf16",
128
+ "nbytes": 33554432,
129
+ "byteOffset": 0
130
+ }
131
+ ],
132
+ "md5sum": "e458ab97e79a42f7d337f26ed979dafa"
133
+ },
134
+ {
135
+ "dataPath": "params_shard_4.bin",
136
+ "format": "raw-shard",
137
+ "nbytes": 25165824,
138
+ "records": [
139
+ {
140
+ "name": "gpt_neox.layers.1.attention.query_key_value.weight",
141
+ "shape": [
142
+ 6144,
143
+ 2048
144
+ ],
145
+ "dtype": "float16",
146
+ "format": "f32-to-bf16",
147
+ "nbytes": 25165824,
148
+ "byteOffset": 0
149
+ }
150
+ ],
151
+ "md5sum": "47dd12a2f3b316282581e8d851f16361"
152
+ },
153
+ {
154
+ "dataPath": "params_shard_5.bin",
155
+ "format": "raw-shard",
156
+ "nbytes": 33554432,
157
+ "records": [
158
+ {
159
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight",
160
+ "shape": [
161
+ 8192,
162
+ 2048
163
+ ],
164
+ "dtype": "float16",
165
+ "format": "f32-to-bf16",
166
+ "nbytes": 33554432,
167
+ "byteOffset": 0
168
+ }
169
+ ],
170
+ "md5sum": "8b376542ee52be1a671247bc19759893"
171
+ },
172
+ {
173
+ "dataPath": "params_shard_6.bin",
174
+ "format": "raw-shard",
175
+ "nbytes": 33554432,
176
+ "records": [
177
+ {
178
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight",
179
+ "shape": [
180
+ 2048,
181
+ 8192
182
+ ],
183
+ "dtype": "float16",
184
+ "format": "f32-to-bf16",
185
+ "nbytes": 33554432,
186
+ "byteOffset": 0
187
+ }
188
+ ],
189
+ "md5sum": "8d429e502c7ffe5efcdeff2840694943"
190
+ },
191
+ {
192
+ "dataPath": "params_shard_7.bin",
193
+ "format": "raw-shard",
194
+ "nbytes": 25165824,
195
+ "records": [
196
+ {
197
+ "name": "gpt_neox.layers.2.attention.query_key_value.weight",
198
+ "shape": [
199
+ 6144,
200
+ 2048
201
+ ],
202
+ "dtype": "float16",
203
+ "format": "f32-to-bf16",
204
+ "nbytes": 25165824,
205
+ "byteOffset": 0
206
+ }
207
+ ],
208
+ "md5sum": "f6b6c9c4ce289692d543d75a0ba5693c"
209
+ },
210
+ {
211
+ "dataPath": "params_shard_8.bin",
212
+ "format": "raw-shard",
213
+ "nbytes": 33554432,
214
+ "records": [
215
+ {
216
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight",
217
+ "shape": [
218
+ 8192,
219
+ 2048
220
+ ],
221
+ "dtype": "float16",
222
+ "format": "f32-to-bf16",
223
+ "nbytes": 33554432,
224
+ "byteOffset": 0
225
+ }
226
+ ],
227
+ "md5sum": "267dc9558c1f41aef67d0e1f022be352"
228
+ },
229
+ {
230
+ "dataPath": "params_shard_9.bin",
231
+ "format": "raw-shard",
232
+ "nbytes": 33554432,
233
+ "records": [
234
+ {
235
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight",
236
+ "shape": [
237
+ 2048,
238
+ 8192
239
+ ],
240
+ "dtype": "float16",
241
+ "format": "f32-to-bf16",
242
+ "nbytes": 33554432,
243
+ "byteOffset": 0
244
+ }
245
+ ],
246
+ "md5sum": "cee8c73decb5ea89ffbef7ee86bf8689"
247
+ },
248
+ {
249
+ "dataPath": "params_shard_10.bin",
250
+ "format": "raw-shard",
251
+ "nbytes": 25165824,
252
+ "records": [
253
+ {
254
+ "name": "gpt_neox.layers.3.attention.query_key_value.weight",
255
+ "shape": [
256
+ 6144,
257
+ 2048
258
+ ],
259
+ "dtype": "float16",
260
+ "format": "f32-to-bf16",
261
+ "nbytes": 25165824,
262
+ "byteOffset": 0
263
+ }
264
+ ],
265
+ "md5sum": "db045065ae5ef84f1464bc836113ef76"
266
+ },
267
+ {
268
+ "dataPath": "params_shard_11.bin",
269
+ "format": "raw-shard",
270
+ "nbytes": 25325568,
271
+ "records": [
272
+ {
273
+ "name": "gpt_neox.layers.0.attention.dense.weight",
274
+ "shape": [
275
+ 2048,
276
+ 2048
277
+ ],
278
+ "dtype": "float16",
279
+ "format": "f32-to-bf16",
280
+ "nbytes": 8388608,
281
+ "byteOffset": 0
282
+ },
283
+ {
284
+ "name": "gpt_neox.layers.0.attention.dense.bias",
285
+ "shape": [
286
+ 2048
287
+ ],
288
+ "dtype": "float16",
289
+ "format": "f32-to-bf16",
290
+ "nbytes": 4096,
291
+ "byteOffset": 8388608
292
+ },
293
+ {
294
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
295
+ "shape": [
296
+ 8192
297
+ ],
298
+ "dtype": "bfloat16",
299
+ "format": "raw",
300
+ "nbytes": 16384,
301
+ "byteOffset": 8392704
302
+ },
303
+ {
304
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
305
+ "shape": [
306
+ 2048
307
+ ],
308
+ "dtype": "bfloat16",
309
+ "format": "raw",
310
+ "nbytes": 4096,
311
+ "byteOffset": 8409088
312
+ },
313
+ {
314
+ "name": "gpt_neox.layers.1.input_layernorm.weight",
315
+ "shape": [
316
+ 2048
317
+ ],
318
+ "dtype": "float16",
319
+ "format": "f32-to-bf16",
320
+ "nbytes": 4096,
321
+ "byteOffset": 8413184
322
+ },
323
+ {
324
+ "name": "gpt_neox.layers.1.input_layernorm.bias",
325
+ "shape": [
326
+ 2048
327
+ ],
328
+ "dtype": "float16",
329
+ "format": "f32-to-bf16",
330
+ "nbytes": 4096,
331
+ "byteOffset": 8417280
332
+ },
333
+ {
334
+ "name": "gpt_neox.layers.1.post_attention_layernorm.weight",
335
+ "shape": [
336
+ 2048
337
+ ],
338
+ "dtype": "float16",
339
+ "format": "f32-to-bf16",
340
+ "nbytes": 4096,
341
+ "byteOffset": 8421376
342
+ },
343
+ {
344
+ "name": "gpt_neox.layers.1.post_attention_layernorm.bias",
345
+ "shape": [
346
+ 2048
347
+ ],
348
+ "dtype": "float16",
349
+ "format": "f32-to-bf16",
350
+ "nbytes": 4096,
351
+ "byteOffset": 8425472
352
+ },
353
+ {
354
+ "name": "gpt_neox.layers.1.attention.query_key_value.bias",
355
+ "shape": [
356
+ 6144
357
+ ],
358
+ "dtype": "float16",
359
+ "format": "f32-to-bf16",
360
+ "nbytes": 12288,
361
+ "byteOffset": 8429568
362
+ },
363
+ {
364
+ "name": "gpt_neox.layers.1.attention.dense.weight",
365
+ "shape": [
366
+ 2048,
367
+ 2048
368
+ ],
369
+ "dtype": "float16",
370
+ "format": "f32-to-bf16",
371
+ "nbytes": 8388608,
372
+ "byteOffset": 8441856
373
+ },
374
+ {
375
+ "name": "gpt_neox.layers.1.attention.dense.bias",
376
+ "shape": [
377
+ 2048
378
+ ],
379
+ "dtype": "float16",
380
+ "format": "f32-to-bf16",
381
+ "nbytes": 4096,
382
+ "byteOffset": 16830464
383
+ },
384
+ {
385
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
386
+ "shape": [
387
+ 8192
388
+ ],
389
+ "dtype": "bfloat16",
390
+ "format": "raw",
391
+ "nbytes": 16384,
392
+ "byteOffset": 16834560
393
+ },
394
+ {
395
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
396
+ "shape": [
397
+ 2048
398
+ ],
399
+ "dtype": "bfloat16",
400
+ "format": "raw",
401
+ "nbytes": 4096,
402
+ "byteOffset": 16850944
403
+ },
404
+ {
405
+ "name": "gpt_neox.layers.2.input_layernorm.weight",
406
+ "shape": [
407
+ 2048
408
+ ],
409
+ "dtype": "float16",
410
+ "format": "f32-to-bf16",
411
+ "nbytes": 4096,
412
+ "byteOffset": 16855040
413
+ },
414
+ {
415
+ "name": "gpt_neox.layers.2.input_layernorm.bias",
416
+ "shape": [
417
+ 2048
418
+ ],
419
+ "dtype": "float16",
420
+ "format": "f32-to-bf16",
421
+ "nbytes": 4096,
422
+ "byteOffset": 16859136
423
+ },
424
+ {
425
+ "name": "gpt_neox.layers.2.post_attention_layernorm.weight",
426
+ "shape": [
427
+ 2048
428
+ ],
429
+ "dtype": "float16",
430
+ "format": "f32-to-bf16",
431
+ "nbytes": 4096,
432
+ "byteOffset": 16863232
433
+ },
434
+ {
435
+ "name": "gpt_neox.layers.2.post_attention_layernorm.bias",
436
+ "shape": [
437
+ 2048
438
+ ],
439
+ "dtype": "float16",
440
+ "format": "f32-to-bf16",
441
+ "nbytes": 4096,
442
+ "byteOffset": 16867328
443
+ },
444
+ {
445
+ "name": "gpt_neox.layers.2.attention.query_key_value.bias",
446
+ "shape": [
447
+ 6144
448
+ ],
449
+ "dtype": "float16",
450
+ "format": "f32-to-bf16",
451
+ "nbytes": 12288,
452
+ "byteOffset": 16871424
453
+ },
454
+ {
455
+ "name": "gpt_neox.layers.2.attention.dense.weight",
456
+ "shape": [
457
+ 2048,
458
+ 2048
459
+ ],
460
+ "dtype": "float16",
461
+ "format": "f32-to-bf16",
462
+ "nbytes": 8388608,
463
+ "byteOffset": 16883712
464
+ },
465
+ {
466
+ "name": "gpt_neox.layers.2.attention.dense.bias",
467
+ "shape": [
468
+ 2048
469
+ ],
470
+ "dtype": "float16",
471
+ "format": "f32-to-bf16",
472
+ "nbytes": 4096,
473
+ "byteOffset": 25272320
474
+ },
475
+ {
476
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
477
+ "shape": [
478
+ 8192
479
+ ],
480
+ "dtype": "bfloat16",
481
+ "format": "raw",
482
+ "nbytes": 16384,
483
+ "byteOffset": 25276416
484
+ },
485
+ {
486
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
487
+ "shape": [
488
+ 2048
489
+ ],
490
+ "dtype": "bfloat16",
491
+ "format": "raw",
492
+ "nbytes": 4096,
493
+ "byteOffset": 25292800
494
+ },
495
+ {
496
+ "name": "gpt_neox.layers.3.input_layernorm.weight",
497
+ "shape": [
498
+ 2048
499
+ ],
500
+ "dtype": "float16",
501
+ "format": "f32-to-bf16",
502
+ "nbytes": 4096,
503
+ "byteOffset": 25296896
504
+ },
505
+ {
506
+ "name": "gpt_neox.layers.3.input_layernorm.bias",
507
+ "shape": [
508
+ 2048
509
+ ],
510
+ "dtype": "float16",
511
+ "format": "f32-to-bf16",
512
+ "nbytes": 4096,
513
+ "byteOffset": 25300992
514
+ },
515
+ {
516
+ "name": "gpt_neox.layers.3.post_attention_layernorm.weight",
517
+ "shape": [
518
+ 2048
519
+ ],
520
+ "dtype": "float16",
521
+ "format": "f32-to-bf16",
522
+ "nbytes": 4096,
523
+ "byteOffset": 25305088
524
+ },
525
+ {
526
+ "name": "gpt_neox.layers.3.post_attention_layernorm.bias",
527
+ "shape": [
528
+ 2048
529
+ ],
530
+ "dtype": "float16",
531
+ "format": "f32-to-bf16",
532
+ "nbytes": 4096,
533
+ "byteOffset": 25309184
534
+ },
535
+ {
536
+ "name": "gpt_neox.layers.3.attention.query_key_value.bias",
537
+ "shape": [
538
+ 6144
539
+ ],
540
+ "dtype": "float16",
541
+ "format": "f32-to-bf16",
542
+ "nbytes": 12288,
543
+ "byteOffset": 25313280
544
+ }
545
+ ],
546
+ "md5sum": "a300f069dd9a566a4d85894be26bd41d"
547
+ },
548
+ {
549
+ "dataPath": "params_shard_12.bin",
550
+ "format": "raw-shard",
551
+ "nbytes": 33554432,
552
+ "records": [
553
+ {
554
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight",
555
+ "shape": [
556
+ 8192,
557
+ 2048
558
+ ],
559
+ "dtype": "float16",
560
+ "format": "f32-to-bf16",
561
+ "nbytes": 33554432,
562
+ "byteOffset": 0
563
+ }
564
+ ],
565
+ "md5sum": "3bf00c3ee5d95ad98f32ea93b3b6ca63"
566
+ },
567
+ {
568
+ "dataPath": "params_shard_13.bin",
569
+ "format": "raw-shard",
570
+ "nbytes": 33554432,
571
+ "records": [
572
+ {
573
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight",
574
+ "shape": [
575
+ 2048,
576
+ 8192
577
+ ],
578
+ "dtype": "float16",
579
+ "format": "f32-to-bf16",
580
+ "nbytes": 33554432,
581
+ "byteOffset": 0
582
+ }
583
+ ],
584
+ "md5sum": "2bc23e26ce74f84d28e0c5f69bf408b3"
585
+ },
586
+ {
587
+ "dataPath": "params_shard_14.bin",
588
+ "format": "raw-shard",
589
+ "nbytes": 25165824,
590
+ "records": [
591
+ {
592
+ "name": "gpt_neox.layers.4.attention.query_key_value.weight",
593
+ "shape": [
594
+ 6144,
595
+ 2048
596
+ ],
597
+ "dtype": "float16",
598
+ "format": "f32-to-bf16",
599
+ "nbytes": 25165824,
600
+ "byteOffset": 0
601
+ }
602
+ ],
603
+ "md5sum": "1aa107f47a4ca56262e01cec478fb5ed"
604
+ },
605
+ {
606
+ "dataPath": "params_shard_15.bin",
607
+ "format": "raw-shard",
608
+ "nbytes": 33554432,
609
+ "records": [
610
+ {
611
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight",
612
+ "shape": [
613
+ 8192,
614
+ 2048
615
+ ],
616
+ "dtype": "float16",
617
+ "format": "f32-to-bf16",
618
+ "nbytes": 33554432,
619
+ "byteOffset": 0
620
+ }
621
+ ],
622
+ "md5sum": "3c65c52252e8eb6acb5856d86e3bc051"
623
+ },
624
+ {
625
+ "dataPath": "params_shard_16.bin",
626
+ "format": "raw-shard",
627
+ "nbytes": 33554432,
628
+ "records": [
629
+ {
630
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight",
631
+ "shape": [
632
+ 2048,
633
+ 8192
634
+ ],
635
+ "dtype": "float16",
636
+ "format": "f32-to-bf16",
637
+ "nbytes": 33554432,
638
+ "byteOffset": 0
639
+ }
640
+ ],
641
+ "md5sum": "311a1252100864f529064f11e902487f"
642
+ },
643
+ {
644
+ "dataPath": "params_shard_17.bin",
645
+ "format": "raw-shard",
646
+ "nbytes": 25165824,
647
+ "records": [
648
+ {
649
+ "name": "gpt_neox.layers.5.attention.query_key_value.weight",
650
+ "shape": [
651
+ 6144,
652
+ 2048
653
+ ],
654
+ "dtype": "float16",
655
+ "format": "f32-to-bf16",
656
+ "nbytes": 25165824,
657
+ "byteOffset": 0
658
+ }
659
+ ],
660
+ "md5sum": "011f5a5c7cb3f338e981f263ed52f81b"
661
+ },
662
+ {
663
+ "dataPath": "params_shard_18.bin",
664
+ "format": "raw-shard",
665
+ "nbytes": 33554432,
666
+ "records": [
667
+ {
668
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight",
669
+ "shape": [
670
+ 8192,
671
+ 2048
672
+ ],
673
+ "dtype": "float16",
674
+ "format": "f32-to-bf16",
675
+ "nbytes": 33554432,
676
+ "byteOffset": 0
677
+ }
678
+ ],
679
+ "md5sum": "df4abbad1146d3fc204e354532df4c32"
680
+ },
681
+ {
682
+ "dataPath": "params_shard_19.bin",
683
+ "format": "raw-shard",
684
+ "nbytes": 33554432,
685
+ "records": [
686
+ {
687
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight",
688
+ "shape": [
689
+ 2048,
690
+ 8192
691
+ ],
692
+ "dtype": "float16",
693
+ "format": "f32-to-bf16",
694
+ "nbytes": 33554432,
695
+ "byteOffset": 0
696
+ }
697
+ ],
698
+ "md5sum": "553c08ac1659b464bf8dba3f262303a1"
699
+ },
700
+ {
701
+ "dataPath": "params_shard_20.bin",
702
+ "format": "raw-shard",
703
+ "nbytes": 25165824,
704
+ "records": [
705
+ {
706
+ "name": "gpt_neox.layers.6.attention.query_key_value.weight",
707
+ "shape": [
708
+ 6144,
709
+ 2048
710
+ ],
711
+ "dtype": "float16",
712
+ "format": "f32-to-bf16",
713
+ "nbytes": 25165824,
714
+ "byteOffset": 0
715
+ }
716
+ ],
717
+ "md5sum": "ba5ea4be88cb85fc9109dac76d309c05"
718
+ },
719
+ {
720
+ "dataPath": "params_shard_21.bin",
721
+ "format": "raw-shard",
722
+ "nbytes": 25325568,
723
+ "records": [
724
+ {
725
+ "name": "gpt_neox.layers.3.attention.dense.weight",
726
+ "shape": [
727
+ 2048,
728
+ 2048
729
+ ],
730
+ "dtype": "float16",
731
+ "format": "f32-to-bf16",
732
+ "nbytes": 8388608,
733
+ "byteOffset": 0
734
+ },
735
+ {
736
+ "name": "gpt_neox.layers.3.attention.dense.bias",
737
+ "shape": [
738
+ 2048
739
+ ],
740
+ "dtype": "float16",
741
+ "format": "f32-to-bf16",
742
+ "nbytes": 4096,
743
+ "byteOffset": 8388608
744
+ },
745
+ {
746
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
747
+ "shape": [
748
+ 8192
749
+ ],
750
+ "dtype": "bfloat16",
751
+ "format": "raw",
752
+ "nbytes": 16384,
753
+ "byteOffset": 8392704
754
+ },
755
+ {
756
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
757
+ "shape": [
758
+ 2048
759
+ ],
760
+ "dtype": "bfloat16",
761
+ "format": "raw",
762
+ "nbytes": 4096,
763
+ "byteOffset": 8409088
764
+ },
765
+ {
766
+ "name": "gpt_neox.layers.4.input_layernorm.weight",
767
+ "shape": [
768
+ 2048
769
+ ],
770
+ "dtype": "float16",
771
+ "format": "f32-to-bf16",
772
+ "nbytes": 4096,
773
+ "byteOffset": 8413184
774
+ },
775
+ {
776
+ "name": "gpt_neox.layers.4.input_layernorm.bias",
777
+ "shape": [
778
+ 2048
779
+ ],
780
+ "dtype": "float16",
781
+ "format": "f32-to-bf16",
782
+ "nbytes": 4096,
783
+ "byteOffset": 8417280
784
+ },
785
+ {
786
+ "name": "gpt_neox.layers.4.post_attention_layernorm.weight",
787
+ "shape": [
788
+ 2048
789
+ ],
790
+ "dtype": "float16",
791
+ "format": "f32-to-bf16",
792
+ "nbytes": 4096,
793
+ "byteOffset": 8421376
794
+ },
795
+ {
796
+ "name": "gpt_neox.layers.4.post_attention_layernorm.bias",
797
+ "shape": [
798
+ 2048
799
+ ],
800
+ "dtype": "float16",
801
+ "format": "f32-to-bf16",
802
+ "nbytes": 4096,
803
+ "byteOffset": 8425472
804
+ },
805
+ {
806
+ "name": "gpt_neox.layers.4.attention.query_key_value.bias",
807
+ "shape": [
808
+ 6144
809
+ ],
810
+ "dtype": "float16",
811
+ "format": "f32-to-bf16",
812
+ "nbytes": 12288,
813
+ "byteOffset": 8429568
814
+ },
815
+ {
816
+ "name": "gpt_neox.layers.4.attention.dense.weight",
817
+ "shape": [
818
+ 2048,
819
+ 2048
820
+ ],
821
+ "dtype": "float16",
822
+ "format": "f32-to-bf16",
823
+ "nbytes": 8388608,
824
+ "byteOffset": 8441856
825
+ },
826
+ {
827
+ "name": "gpt_neox.layers.4.attention.dense.bias",
828
+ "shape": [
829
+ 2048
830
+ ],
831
+ "dtype": "float16",
832
+ "format": "f32-to-bf16",
833
+ "nbytes": 4096,
834
+ "byteOffset": 16830464
835
+ },
836
+ {
837
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
838
+ "shape": [
839
+ 8192
840
+ ],
841
+ "dtype": "bfloat16",
842
+ "format": "raw",
843
+ "nbytes": 16384,
844
+ "byteOffset": 16834560
845
+ },
846
+ {
847
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
848
+ "shape": [
849
+ 2048
850
+ ],
851
+ "dtype": "bfloat16",
852
+ "format": "raw",
853
+ "nbytes": 4096,
854
+ "byteOffset": 16850944
855
+ },
856
+ {
857
+ "name": "gpt_neox.layers.5.input_layernorm.weight",
858
+ "shape": [
859
+ 2048
860
+ ],
861
+ "dtype": "float16",
862
+ "format": "f32-to-bf16",
863
+ "nbytes": 4096,
864
+ "byteOffset": 16855040
865
+ },
866
+ {
867
+ "name": "gpt_neox.layers.5.input_layernorm.bias",
868
+ "shape": [
869
+ 2048
870
+ ],
871
+ "dtype": "float16",
872
+ "format": "f32-to-bf16",
873
+ "nbytes": 4096,
874
+ "byteOffset": 16859136
875
+ },
876
+ {
877
+ "name": "gpt_neox.layers.5.post_attention_layernorm.weight",
878
+ "shape": [
879
+ 2048
880
+ ],
881
+ "dtype": "float16",
882
+ "format": "f32-to-bf16",
883
+ "nbytes": 4096,
884
+ "byteOffset": 16863232
885
+ },
886
+ {
887
+ "name": "gpt_neox.layers.5.post_attention_layernorm.bias",
888
+ "shape": [
889
+ 2048
890
+ ],
891
+ "dtype": "float16",
892
+ "format": "f32-to-bf16",
893
+ "nbytes": 4096,
894
+ "byteOffset": 16867328
895
+ },
896
+ {
897
+ "name": "gpt_neox.layers.5.attention.query_key_value.bias",
898
+ "shape": [
899
+ 6144
900
+ ],
901
+ "dtype": "float16",
902
+ "format": "f32-to-bf16",
903
+ "nbytes": 12288,
904
+ "byteOffset": 16871424
905
+ },
906
+ {
907
+ "name": "gpt_neox.layers.5.attention.dense.weight",
908
+ "shape": [
909
+ 2048,
910
+ 2048
911
+ ],
912
+ "dtype": "float16",
913
+ "format": "f32-to-bf16",
914
+ "nbytes": 8388608,
915
+ "byteOffset": 16883712
916
+ },
917
+ {
918
+ "name": "gpt_neox.layers.5.attention.dense.bias",
919
+ "shape": [
920
+ 2048
921
+ ],
922
+ "dtype": "float16",
923
+ "format": "f32-to-bf16",
924
+ "nbytes": 4096,
925
+ "byteOffset": 25272320
926
+ },
927
+ {
928
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
929
+ "shape": [
930
+ 8192
931
+ ],
932
+ "dtype": "bfloat16",
933
+ "format": "raw",
934
+ "nbytes": 16384,
935
+ "byteOffset": 25276416
936
+ },
937
+ {
938
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
939
+ "shape": [
940
+ 2048
941
+ ],
942
+ "dtype": "bfloat16",
943
+ "format": "raw",
944
+ "nbytes": 4096,
945
+ "byteOffset": 25292800
946
+ },
947
+ {
948
+ "name": "gpt_neox.layers.6.input_layernorm.weight",
949
+ "shape": [
950
+ 2048
951
+ ],
952
+ "dtype": "float16",
953
+ "format": "f32-to-bf16",
954
+ "nbytes": 4096,
955
+ "byteOffset": 25296896
956
+ },
957
+ {
958
+ "name": "gpt_neox.layers.6.input_layernorm.bias",
959
+ "shape": [
960
+ 2048
961
+ ],
962
+ "dtype": "float16",
963
+ "format": "f32-to-bf16",
964
+ "nbytes": 4096,
965
+ "byteOffset": 25300992
966
+ },
967
+ {
968
+ "name": "gpt_neox.layers.6.post_attention_layernorm.weight",
969
+ "shape": [
970
+ 2048
971
+ ],
972
+ "dtype": "float16",
973
+ "format": "f32-to-bf16",
974
+ "nbytes": 4096,
975
+ "byteOffset": 25305088
976
+ },
977
+ {
978
+ "name": "gpt_neox.layers.6.post_attention_layernorm.bias",
979
+ "shape": [
980
+ 2048
981
+ ],
982
+ "dtype": "float16",
983
+ "format": "f32-to-bf16",
984
+ "nbytes": 4096,
985
+ "byteOffset": 25309184
986
+ },
987
+ {
988
+ "name": "gpt_neox.layers.6.attention.query_key_value.bias",
989
+ "shape": [
990
+ 6144
991
+ ],
992
+ "dtype": "float16",
993
+ "format": "f32-to-bf16",
994
+ "nbytes": 12288,
995
+ "byteOffset": 25313280
996
+ }
997
+ ],
998
+ "md5sum": "0f848458ed7ba85e561c6ac6953346aa"
999
+ },
1000
+ {
1001
+ "dataPath": "params_shard_22.bin",
1002
+ "format": "raw-shard",
1003
+ "nbytes": 33554432,
1004
+ "records": [
1005
+ {
1006
+ "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.weight",
1007
+ "shape": [
1008
+ 8192,
1009
+ 2048
1010
+ ],
1011
+ "dtype": "float16",
1012
+ "format": "f32-to-bf16",
1013
+ "nbytes": 33554432,
1014
+ "byteOffset": 0
1015
+ }
1016
+ ],
1017
+ "md5sum": "c483f21a9a023e8537dcb3a97635ac32"
1018
+ },
1019
+ {
1020
+ "dataPath": "params_shard_23.bin",
1021
+ "format": "raw-shard",
1022
+ "nbytes": 33554432,
1023
+ "records": [
1024
+ {
1025
+ "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.weight",
1026
+ "shape": [
1027
+ 2048,
1028
+ 8192
1029
+ ],
1030
+ "dtype": "float16",
1031
+ "format": "f32-to-bf16",
1032
+ "nbytes": 33554432,
1033
+ "byteOffset": 0
1034
+ }
1035
+ ],
1036
+ "md5sum": "1b8f8a2187e8fe0821a62f6aecc43903"
1037
+ },
1038
+ {
1039
+ "dataPath": "params_shard_24.bin",
1040
+ "format": "raw-shard",
1041
+ "nbytes": 25165824,
1042
+ "records": [
1043
+ {
1044
+ "name": "gpt_neox.layers.7.attention.query_key_value.weight",
1045
+ "shape": [
1046
+ 6144,
1047
+ 2048
1048
+ ],
1049
+ "dtype": "float16",
1050
+ "format": "f32-to-bf16",
1051
+ "nbytes": 25165824,
1052
+ "byteOffset": 0
1053
+ }
1054
+ ],
1055
+ "md5sum": "0c3a014c3c16ecf900cabc943c57a8b4"
1056
+ },
1057
+ {
1058
+ "dataPath": "params_shard_25.bin",
1059
+ "format": "raw-shard",
1060
+ "nbytes": 33554432,
1061
+ "records": [
1062
+ {
1063
+ "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.weight",
1064
+ "shape": [
1065
+ 8192,
1066
+ 2048
1067
+ ],
1068
+ "dtype": "float16",
1069
+ "format": "f32-to-bf16",
1070
+ "nbytes": 33554432,
1071
+ "byteOffset": 0
1072
+ }
1073
+ ],
1074
+ "md5sum": "3a068195dd649252ba4aab5ebff92b9c"
1075
+ },
1076
+ {
1077
+ "dataPath": "params_shard_26.bin",
1078
+ "format": "raw-shard",
1079
+ "nbytes": 33554432,
1080
+ "records": [
1081
+ {
1082
+ "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.weight",
1083
+ "shape": [
1084
+ 2048,
1085
+ 8192
1086
+ ],
1087
+ "dtype": "float16",
1088
+ "format": "f32-to-bf16",
1089
+ "nbytes": 33554432,
1090
+ "byteOffset": 0
1091
+ }
1092
+ ],
1093
+ "md5sum": "c32830c1ad07599365b35bdb97040713"
1094
+ },
1095
+ {
1096
+ "dataPath": "params_shard_27.bin",
1097
+ "format": "raw-shard",
1098
+ "nbytes": 25165824,
1099
+ "records": [
1100
+ {
1101
+ "name": "gpt_neox.layers.8.attention.query_key_value.weight",
1102
+ "shape": [
1103
+ 6144,
1104
+ 2048
1105
+ ],
1106
+ "dtype": "float16",
1107
+ "format": "f32-to-bf16",
1108
+ "nbytes": 25165824,
1109
+ "byteOffset": 0
1110
+ }
1111
+ ],
1112
+ "md5sum": "517ecd0e8410e75bd7b3be0f44d4f656"
1113
+ },
1114
+ {
1115
+ "dataPath": "params_shard_28.bin",
1116
+ "format": "raw-shard",
1117
+ "nbytes": 33554432,
1118
+ "records": [
1119
+ {
1120
+ "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.weight",
1121
+ "shape": [
1122
+ 8192,
1123
+ 2048
1124
+ ],
1125
+ "dtype": "float16",
1126
+ "format": "f32-to-bf16",
1127
+ "nbytes": 33554432,
1128
+ "byteOffset": 0
1129
+ }
1130
+ ],
1131
+ "md5sum": "4513e52799f3966b3090fe8f2c198570"
1132
+ },
1133
+ {
1134
+ "dataPath": "params_shard_29.bin",
1135
+ "format": "raw-shard",
1136
+ "nbytes": 33554432,
1137
+ "records": [
1138
+ {
1139
+ "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.weight",
1140
+ "shape": [
1141
+ 2048,
1142
+ 8192
1143
+ ],
1144
+ "dtype": "float16",
1145
+ "format": "f32-to-bf16",
1146
+ "nbytes": 33554432,
1147
+ "byteOffset": 0
1148
+ }
1149
+ ],
1150
+ "md5sum": "20f134a1a67f8da542a1594f8d145687"
1151
+ },
1152
+ {
1153
+ "dataPath": "params_shard_30.bin",
1154
+ "format": "raw-shard",
1155
+ "nbytes": 25165824,
1156
+ "records": [
1157
+ {
1158
+ "name": "gpt_neox.layers.9.attention.query_key_value.weight",
1159
+ "shape": [
1160
+ 6144,
1161
+ 2048
1162
+ ],
1163
+ "dtype": "float16",
1164
+ "format": "f32-to-bf16",
1165
+ "nbytes": 25165824,
1166
+ "byteOffset": 0
1167
+ }
1168
+ ],
1169
+ "md5sum": "7cda7ef64a1611eb36cff4f88fa70fd7"
1170
+ },
1171
+ {
1172
+ "dataPath": "params_shard_31.bin",
1173
+ "format": "raw-shard",
1174
+ "nbytes": 25325568,
1175
+ "records": [
1176
+ {
1177
+ "name": "gpt_neox.layers.6.attention.dense.weight",
1178
+ "shape": [
1179
+ 2048,
1180
+ 2048
1181
+ ],
1182
+ "dtype": "float16",
1183
+ "format": "f32-to-bf16",
1184
+ "nbytes": 8388608,
1185
+ "byteOffset": 0
1186
+ },
1187
+ {
1188
+ "name": "gpt_neox.layers.6.attention.dense.bias",
1189
+ "shape": [
1190
+ 2048
1191
+ ],
1192
+ "dtype": "float16",
1193
+ "format": "f32-to-bf16",
1194
+ "nbytes": 4096,
1195
+ "byteOffset": 8388608
1196
+ },
1197
+ {
1198
+ "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
1199
+ "shape": [
1200
+ 8192
1201
+ ],
1202
+ "dtype": "bfloat16",
1203
+ "format": "raw",
1204
+ "nbytes": 16384,
1205
+ "byteOffset": 8392704
1206
+ },
1207
+ {
1208
+ "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
1209
+ "shape": [
1210
+ 2048
1211
+ ],
1212
+ "dtype": "bfloat16",
1213
+ "format": "raw",
1214
+ "nbytes": 4096,
1215
+ "byteOffset": 8409088
1216
+ },
1217
+ {
1218
+ "name": "gpt_neox.layers.7.input_layernorm.weight",
1219
+ "shape": [
1220
+ 2048
1221
+ ],
1222
+ "dtype": "float16",
1223
+ "format": "f32-to-bf16",
1224
+ "nbytes": 4096,
1225
+ "byteOffset": 8413184
1226
+ },
1227
+ {
1228
+ "name": "gpt_neox.layers.7.input_layernorm.bias",
1229
+ "shape": [
1230
+ 2048
1231
+ ],
1232
+ "dtype": "float16",
1233
+ "format": "f32-to-bf16",
1234
+ "nbytes": 4096,
1235
+ "byteOffset": 8417280
1236
+ },
1237
+ {
1238
+ "name": "gpt_neox.layers.7.post_attention_layernorm.weight",
1239
+ "shape": [
1240
+ 2048
1241
+ ],
1242
+ "dtype": "float16",
1243
+ "format": "f32-to-bf16",
1244
+ "nbytes": 4096,
1245
+ "byteOffset": 8421376
1246
+ },
1247
+ {
1248
+ "name": "gpt_neox.layers.7.post_attention_layernorm.bias",
1249
+ "shape": [
1250
+ 2048
1251
+ ],
1252
+ "dtype": "float16",
1253
+ "format": "f32-to-bf16",
1254
+ "nbytes": 4096,
1255
+ "byteOffset": 8425472
1256
+ },
1257
+ {
1258
+ "name": "gpt_neox.layers.7.attention.query_key_value.bias",
1259
+ "shape": [
1260
+ 6144
1261
+ ],
1262
+ "dtype": "float16",
1263
+ "format": "f32-to-bf16",
1264
+ "nbytes": 12288,
1265
+ "byteOffset": 8429568
1266
+ },
1267
+ {
1268
+ "name": "gpt_neox.layers.7.attention.dense.weight",
1269
+ "shape": [
1270
+ 2048,
1271
+ 2048
1272
+ ],
1273
+ "dtype": "float16",
1274
+ "format": "f32-to-bf16",
1275
+ "nbytes": 8388608,
1276
+ "byteOffset": 8441856
1277
+ },
1278
+ {
1279
+ "name": "gpt_neox.layers.7.attention.dense.bias",
1280
+ "shape": [
1281
+ 2048
1282
+ ],
1283
+ "dtype": "float16",
1284
+ "format": "f32-to-bf16",
1285
+ "nbytes": 4096,
1286
+ "byteOffset": 16830464
1287
+ },
1288
+ {
1289
+ "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
1290
+ "shape": [
1291
+ 8192
1292
+ ],
1293
+ "dtype": "bfloat16",
1294
+ "format": "raw",
1295
+ "nbytes": 16384,
1296
+ "byteOffset": 16834560
1297
+ },
1298
+ {
1299
+ "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
1300
+ "shape": [
1301
+ 2048
1302
+ ],
1303
+ "dtype": "bfloat16",
1304
+ "format": "raw",
1305
+ "nbytes": 4096,
1306
+ "byteOffset": 16850944
1307
+ },
1308
+ {
1309
+ "name": "gpt_neox.layers.8.input_layernorm.weight",
1310
+ "shape": [
1311
+ 2048
1312
+ ],
1313
+ "dtype": "float16",
1314
+ "format": "f32-to-bf16",
1315
+ "nbytes": 4096,
1316
+ "byteOffset": 16855040
1317
+ },
1318
+ {
1319
+ "name": "gpt_neox.layers.8.input_layernorm.bias",
1320
+ "shape": [
1321
+ 2048
1322
+ ],
1323
+ "dtype": "float16",
1324
+ "format": "f32-to-bf16",
1325
+ "nbytes": 4096,
1326
+ "byteOffset": 16859136
1327
+ },
1328
+ {
1329
+ "name": "gpt_neox.layers.8.post_attention_layernorm.weight",
1330
+ "shape": [
1331
+ 2048
1332
+ ],
1333
+ "dtype": "float16",
1334
+ "format": "f32-to-bf16",
1335
+ "nbytes": 4096,
1336
+ "byteOffset": 16863232
1337
+ },
1338
+ {
1339
+ "name": "gpt_neox.layers.8.post_attention_layernorm.bias",
1340
+ "shape": [
1341
+ 2048
1342
+ ],
1343
+ "dtype": "float16",
1344
+ "format": "f32-to-bf16",
1345
+ "nbytes": 4096,
1346
+ "byteOffset": 16867328
1347
+ },
1348
+ {
1349
+ "name": "gpt_neox.layers.8.attention.query_key_value.bias",
1350
+ "shape": [
1351
+ 6144
1352
+ ],
1353
+ "dtype": "float16",
1354
+ "format": "f32-to-bf16",
1355
+ "nbytes": 12288,
1356
+ "byteOffset": 16871424
1357
+ },
1358
+ {
1359
+ "name": "gpt_neox.layers.8.attention.dense.weight",
1360
+ "shape": [
1361
+ 2048,
1362
+ 2048
1363
+ ],
1364
+ "dtype": "float16",
1365
+ "format": "f32-to-bf16",
1366
+ "nbytes": 8388608,
1367
+ "byteOffset": 16883712
1368
+ },
1369
+ {
1370
+ "name": "gpt_neox.layers.8.attention.dense.bias",
1371
+ "shape": [
1372
+ 2048
1373
+ ],
1374
+ "dtype": "float16",
1375
+ "format": "f32-to-bf16",
1376
+ "nbytes": 4096,
1377
+ "byteOffset": 25272320
1378
+ },
1379
+ {
1380
+ "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
1381
+ "shape": [
1382
+ 8192
1383
+ ],
1384
+ "dtype": "bfloat16",
1385
+ "format": "raw",
1386
+ "nbytes": 16384,
1387
+ "byteOffset": 25276416
1388
+ },
1389
+ {
1390
+ "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
1391
+ "shape": [
1392
+ 2048
1393
+ ],
1394
+ "dtype": "bfloat16",
1395
+ "format": "raw",
1396
+ "nbytes": 4096,
1397
+ "byteOffset": 25292800
1398
+ },
1399
+ {
1400
+ "name": "gpt_neox.layers.9.input_layernorm.weight",
1401
+ "shape": [
1402
+ 2048
1403
+ ],
1404
+ "dtype": "float16",
1405
+ "format": "f32-to-bf16",
1406
+ "nbytes": 4096,
1407
+ "byteOffset": 25296896
1408
+ },
1409
+ {
1410
+ "name": "gpt_neox.layers.9.input_layernorm.bias",
1411
+ "shape": [
1412
+ 2048
1413
+ ],
1414
+ "dtype": "float16",
1415
+ "format": "f32-to-bf16",
1416
+ "nbytes": 4096,
1417
+ "byteOffset": 25300992
1418
+ },
1419
+ {
1420
+ "name": "gpt_neox.layers.9.post_attention_layernorm.weight",
1421
+ "shape": [
1422
+ 2048
1423
+ ],
1424
+ "dtype": "float16",
1425
+ "format": "f32-to-bf16",
1426
+ "nbytes": 4096,
1427
+ "byteOffset": 25305088
1428
+ },
1429
+ {
1430
+ "name": "gpt_neox.layers.9.post_attention_layernorm.bias",
1431
+ "shape": [
1432
+ 2048
1433
+ ],
1434
+ "dtype": "float16",
1435
+ "format": "f32-to-bf16",
1436
+ "nbytes": 4096,
1437
+ "byteOffset": 25309184
1438
+ },
1439
+ {
1440
+ "name": "gpt_neox.layers.9.attention.query_key_value.bias",
1441
+ "shape": [
1442
+ 6144
1443
+ ],
1444
+ "dtype": "float16",
1445
+ "format": "f32-to-bf16",
1446
+ "nbytes": 12288,
1447
+ "byteOffset": 25313280
1448
+ }
1449
+ ],
1450
+ "md5sum": "e259ef76f92def9684153bc75388edae"
1451
+ },
1452
+ {
1453
+ "dataPath": "params_shard_32.bin",
1454
+ "format": "raw-shard",
1455
+ "nbytes": 33554432,
1456
+ "records": [
1457
+ {
1458
+ "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.weight",
1459
+ "shape": [
1460
+ 8192,
1461
+ 2048
1462
+ ],
1463
+ "dtype": "float16",
1464
+ "format": "f32-to-bf16",
1465
+ "nbytes": 33554432,
1466
+ "byteOffset": 0
1467
+ }
1468
+ ],
1469
+ "md5sum": "4e2fa9d7b430b8093610199bacc9e7a8"
1470
+ },
1471
+ {
1472
+ "dataPath": "params_shard_33.bin",
1473
+ "format": "raw-shard",
1474
+ "nbytes": 33554432,
1475
+ "records": [
1476
+ {
1477
+ "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.weight",
1478
+ "shape": [
1479
+ 2048,
1480
+ 8192
1481
+ ],
1482
+ "dtype": "float16",
1483
+ "format": "f32-to-bf16",
1484
+ "nbytes": 33554432,
1485
+ "byteOffset": 0
1486
+ }
1487
+ ],
1488
+ "md5sum": "833726affdd62d945fea452daa4be81e"
1489
+ },
1490
+ {
1491
+ "dataPath": "params_shard_34.bin",
1492
+ "format": "raw-shard",
1493
+ "nbytes": 25165824,
1494
+ "records": [
1495
+ {
1496
+ "name": "gpt_neox.layers.10.attention.query_key_value.weight",
1497
+ "shape": [
1498
+ 6144,
1499
+ 2048
1500
+ ],
1501
+ "dtype": "float16",
1502
+ "format": "f32-to-bf16",
1503
+ "nbytes": 25165824,
1504
+ "byteOffset": 0
1505
+ }
1506
+ ],
1507
+ "md5sum": "a90521567e0f31a50ab8ee830a33ec11"
1508
+ },
1509
+ {
1510
+ "dataPath": "params_shard_35.bin",
1511
+ "format": "raw-shard",
1512
+ "nbytes": 33554432,
1513
+ "records": [
1514
+ {
1515
+ "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.weight",
1516
+ "shape": [
1517
+ 8192,
1518
+ 2048
1519
+ ],
1520
+ "dtype": "float16",
1521
+ "format": "f32-to-bf16",
1522
+ "nbytes": 33554432,
1523
+ "byteOffset": 0
1524
+ }
1525
+ ],
1526
+ "md5sum": "2017f8b0fcce69d68a36eddf4f065fb6"
1527
+ },
1528
+ {
1529
+ "dataPath": "params_shard_36.bin",
1530
+ "format": "raw-shard",
1531
+ "nbytes": 33554432,
1532
+ "records": [
1533
+ {
1534
+ "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.weight",
1535
+ "shape": [
1536
+ 2048,
1537
+ 8192
1538
+ ],
1539
+ "dtype": "float16",
1540
+ "format": "f32-to-bf16",
1541
+ "nbytes": 33554432,
1542
+ "byteOffset": 0
1543
+ }
1544
+ ],
1545
+ "md5sum": "1bb0df0f78e29b562fb80e9416f2e6d0"
1546
+ },
1547
+ {
1548
+ "dataPath": "params_shard_37.bin",
1549
+ "format": "raw-shard",
1550
+ "nbytes": 25165824,
1551
+ "records": [
1552
+ {
1553
+ "name": "gpt_neox.layers.11.attention.query_key_value.weight",
1554
+ "shape": [
1555
+ 6144,
1556
+ 2048
1557
+ ],
1558
+ "dtype": "float16",
1559
+ "format": "f32-to-bf16",
1560
+ "nbytes": 25165824,
1561
+ "byteOffset": 0
1562
+ }
1563
+ ],
1564
+ "md5sum": "c5bcae6dd378facfbb316629a353996e"
1565
+ },
1566
+ {
1567
+ "dataPath": "params_shard_38.bin",
1568
+ "format": "raw-shard",
1569
+ "nbytes": 33554432,
1570
+ "records": [
1571
+ {
1572
+ "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.weight",
1573
+ "shape": [
1574
+ 8192,
1575
+ 2048
1576
+ ],
1577
+ "dtype": "float16",
1578
+ "format": "f32-to-bf16",
1579
+ "nbytes": 33554432,
1580
+ "byteOffset": 0
1581
+ }
1582
+ ],
1583
+ "md5sum": "77116769d32f06f166da36ca2a268c64"
1584
+ },
1585
+ {
1586
+ "dataPath": "params_shard_39.bin",
1587
+ "format": "raw-shard",
1588
+ "nbytes": 33554432,
1589
+ "records": [
1590
+ {
1591
+ "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.weight",
1592
+ "shape": [
1593
+ 2048,
1594
+ 8192
1595
+ ],
1596
+ "dtype": "float16",
1597
+ "format": "f32-to-bf16",
1598
+ "nbytes": 33554432,
1599
+ "byteOffset": 0
1600
+ }
1601
+ ],
1602
+ "md5sum": "b9e187b32f191e5cd83521eb59d918fd"
1603
+ },
1604
+ {
1605
+ "dataPath": "params_shard_40.bin",
1606
+ "format": "raw-shard",
1607
+ "nbytes": 25165824,
1608
+ "records": [
1609
+ {
1610
+ "name": "gpt_neox.layers.12.attention.query_key_value.weight",
1611
+ "shape": [
1612
+ 6144,
1613
+ 2048
1614
+ ],
1615
+ "dtype": "float16",
1616
+ "format": "f32-to-bf16",
1617
+ "nbytes": 25165824,
1618
+ "byteOffset": 0
1619
+ }
1620
+ ],
1621
+ "md5sum": "010608c5a52ef822e207474b2f209681"
1622
+ },
1623
+ {
1624
+ "dataPath": "params_shard_41.bin",
1625
+ "format": "raw-shard",
1626
+ "nbytes": 25325568,
1627
+ "records": [
1628
+ {
1629
+ "name": "gpt_neox.layers.9.attention.dense.weight",
1630
+ "shape": [
1631
+ 2048,
1632
+ 2048
1633
+ ],
1634
+ "dtype": "float16",
1635
+ "format": "f32-to-bf16",
1636
+ "nbytes": 8388608,
1637
+ "byteOffset": 0
1638
+ },
1639
+ {
1640
+ "name": "gpt_neox.layers.9.attention.dense.bias",
1641
+ "shape": [
1642
+ 2048
1643
+ ],
1644
+ "dtype": "float16",
1645
+ "format": "f32-to-bf16",
1646
+ "nbytes": 4096,
1647
+ "byteOffset": 8388608
1648
+ },
1649
+ {
1650
+ "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
1651
+ "shape": [
1652
+ 8192
1653
+ ],
1654
+ "dtype": "bfloat16",
1655
+ "format": "raw",
1656
+ "nbytes": 16384,
1657
+ "byteOffset": 8392704
1658
+ },
1659
+ {
1660
+ "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
1661
+ "shape": [
1662
+ 2048
1663
+ ],
1664
+ "dtype": "bfloat16",
1665
+ "format": "raw",
1666
+ "nbytes": 4096,
1667
+ "byteOffset": 8409088
1668
+ },
1669
+ {
1670
+ "name": "gpt_neox.layers.10.input_layernorm.weight",
1671
+ "shape": [
1672
+ 2048
1673
+ ],
1674
+ "dtype": "float16",
1675
+ "format": "f32-to-bf16",
1676
+ "nbytes": 4096,
1677
+ "byteOffset": 8413184
1678
+ },
1679
+ {
1680
+ "name": "gpt_neox.layers.10.input_layernorm.bias",
1681
+ "shape": [
1682
+ 2048
1683
+ ],
1684
+ "dtype": "float16",
1685
+ "format": "f32-to-bf16",
1686
+ "nbytes": 4096,
1687
+ "byteOffset": 8417280
1688
+ },
1689
+ {
1690
+ "name": "gpt_neox.layers.10.post_attention_layernorm.weight",
1691
+ "shape": [
1692
+ 2048
1693
+ ],
1694
+ "dtype": "float16",
1695
+ "format": "f32-to-bf16",
1696
+ "nbytes": 4096,
1697
+ "byteOffset": 8421376
1698
+ },
1699
+ {
1700
+ "name": "gpt_neox.layers.10.post_attention_layernorm.bias",
1701
+ "shape": [
1702
+ 2048
1703
+ ],
1704
+ "dtype": "float16",
1705
+ "format": "f32-to-bf16",
1706
+ "nbytes": 4096,
1707
+ "byteOffset": 8425472
1708
+ },
1709
+ {
1710
+ "name": "gpt_neox.layers.10.attention.query_key_value.bias",
1711
+ "shape": [
1712
+ 6144
1713
+ ],
1714
+ "dtype": "float16",
1715
+ "format": "f32-to-bf16",
1716
+ "nbytes": 12288,
1717
+ "byteOffset": 8429568
1718
+ },
1719
+ {
1720
+ "name": "gpt_neox.layers.10.attention.dense.weight",
1721
+ "shape": [
1722
+ 2048,
1723
+ 2048
1724
+ ],
1725
+ "dtype": "float16",
1726
+ "format": "f32-to-bf16",
1727
+ "nbytes": 8388608,
1728
+ "byteOffset": 8441856
1729
+ },
1730
+ {
1731
+ "name": "gpt_neox.layers.10.attention.dense.bias",
1732
+ "shape": [
1733
+ 2048
1734
+ ],
1735
+ "dtype": "float16",
1736
+ "format": "f32-to-bf16",
1737
+ "nbytes": 4096,
1738
+ "byteOffset": 16830464
1739
+ },
1740
+ {
1741
+ "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
1742
+ "shape": [
1743
+ 8192
1744
+ ],
1745
+ "dtype": "bfloat16",
1746
+ "format": "raw",
1747
+ "nbytes": 16384,
1748
+ "byteOffset": 16834560
1749
+ },
1750
+ {
1751
+ "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
1752
+ "shape": [
1753
+ 2048
1754
+ ],
1755
+ "dtype": "bfloat16",
1756
+ "format": "raw",
1757
+ "nbytes": 4096,
1758
+ "byteOffset": 16850944
1759
+ },
1760
+ {
1761
+ "name": "gpt_neox.layers.11.input_layernorm.weight",
1762
+ "shape": [
1763
+ 2048
1764
+ ],
1765
+ "dtype": "float16",
1766
+ "format": "f32-to-bf16",
1767
+ "nbytes": 4096,
1768
+ "byteOffset": 16855040
1769
+ },
1770
+ {
1771
+ "name": "gpt_neox.layers.11.input_layernorm.bias",
1772
+ "shape": [
1773
+ 2048
1774
+ ],
1775
+ "dtype": "float16",
1776
+ "format": "f32-to-bf16",
1777
+ "nbytes": 4096,
1778
+ "byteOffset": 16859136
1779
+ },
1780
+ {
1781
+ "name": "gpt_neox.layers.11.post_attention_layernorm.weight",
1782
+ "shape": [
1783
+ 2048
1784
+ ],
1785
+ "dtype": "float16",
1786
+ "format": "f32-to-bf16",
1787
+ "nbytes": 4096,
1788
+ "byteOffset": 16863232
1789
+ },
1790
+ {
1791
+ "name": "gpt_neox.layers.11.post_attention_layernorm.bias",
1792
+ "shape": [
1793
+ 2048
1794
+ ],
1795
+ "dtype": "float16",
1796
+ "format": "f32-to-bf16",
1797
+ "nbytes": 4096,
1798
+ "byteOffset": 16867328
1799
+ },
1800
+ {
1801
+ "name": "gpt_neox.layers.11.attention.query_key_value.bias",
1802
+ "shape": [
1803
+ 6144
1804
+ ],
1805
+ "dtype": "float16",
1806
+ "format": "f32-to-bf16",
1807
+ "nbytes": 12288,
1808
+ "byteOffset": 16871424
1809
+ },
1810
+ {
1811
+ "name": "gpt_neox.layers.11.attention.dense.weight",
1812
+ "shape": [
1813
+ 2048,
1814
+ 2048
1815
+ ],
1816
+ "dtype": "float16",
1817
+ "format": "f32-to-bf16",
1818
+ "nbytes": 8388608,
1819
+ "byteOffset": 16883712
1820
+ },
1821
+ {
1822
+ "name": "gpt_neox.layers.11.attention.dense.bias",
1823
+ "shape": [
1824
+ 2048
1825
+ ],
1826
+ "dtype": "float16",
1827
+ "format": "f32-to-bf16",
1828
+ "nbytes": 4096,
1829
+ "byteOffset": 25272320
1830
+ },
1831
+ {
1832
+ "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
1833
+ "shape": [
1834
+ 8192
1835
+ ],
1836
+ "dtype": "bfloat16",
1837
+ "format": "raw",
1838
+ "nbytes": 16384,
1839
+ "byteOffset": 25276416
1840
+ },
1841
+ {
1842
+ "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
1843
+ "shape": [
1844
+ 2048
1845
+ ],
1846
+ "dtype": "bfloat16",
1847
+ "format": "raw",
1848
+ "nbytes": 4096,
1849
+ "byteOffset": 25292800
1850
+ },
1851
+ {
1852
+ "name": "gpt_neox.layers.12.input_layernorm.weight",
1853
+ "shape": [
1854
+ 2048
1855
+ ],
1856
+ "dtype": "float16",
1857
+ "format": "f32-to-bf16",
1858
+ "nbytes": 4096,
1859
+ "byteOffset": 25296896
1860
+ },
1861
+ {
1862
+ "name": "gpt_neox.layers.12.input_layernorm.bias",
1863
+ "shape": [
1864
+ 2048
1865
+ ],
1866
+ "dtype": "float16",
1867
+ "format": "f32-to-bf16",
1868
+ "nbytes": 4096,
1869
+ "byteOffset": 25300992
1870
+ },
1871
+ {
1872
+ "name": "gpt_neox.layers.12.post_attention_layernorm.weight",
1873
+ "shape": [
1874
+ 2048
1875
+ ],
1876
+ "dtype": "float16",
1877
+ "format": "f32-to-bf16",
1878
+ "nbytes": 4096,
1879
+ "byteOffset": 25305088
1880
+ },
1881
+ {
1882
+ "name": "gpt_neox.layers.12.post_attention_layernorm.bias",
1883
+ "shape": [
1884
+ 2048
1885
+ ],
1886
+ "dtype": "float16",
1887
+ "format": "f32-to-bf16",
1888
+ "nbytes": 4096,
1889
+ "byteOffset": 25309184
1890
+ },
1891
+ {
1892
+ "name": "gpt_neox.layers.12.attention.query_key_value.bias",
1893
+ "shape": [
1894
+ 6144
1895
+ ],
1896
+ "dtype": "float16",
1897
+ "format": "f32-to-bf16",
1898
+ "nbytes": 12288,
1899
+ "byteOffset": 25313280
1900
+ }
1901
+ ],
1902
+ "md5sum": "9d0c8914ae6f3f6fcfb66ac13eb4d0a3"
1903
+ },
1904
+ {
1905
+ "dataPath": "params_shard_42.bin",
1906
+ "format": "raw-shard",
1907
+ "nbytes": 33554432,
1908
+ "records": [
1909
+ {
1910
+ "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.weight",
1911
+ "shape": [
1912
+ 8192,
1913
+ 2048
1914
+ ],
1915
+ "dtype": "float16",
1916
+ "format": "f32-to-bf16",
1917
+ "nbytes": 33554432,
1918
+ "byteOffset": 0
1919
+ }
1920
+ ],
1921
+ "md5sum": "ab87a989c27431535f15efa4e031aad7"
1922
+ },
1923
+ {
1924
+ "dataPath": "params_shard_43.bin",
1925
+ "format": "raw-shard",
1926
+ "nbytes": 33554432,
1927
+ "records": [
1928
+ {
1929
+ "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.weight",
1930
+ "shape": [
1931
+ 2048,
1932
+ 8192
1933
+ ],
1934
+ "dtype": "float16",
1935
+ "format": "f32-to-bf16",
1936
+ "nbytes": 33554432,
1937
+ "byteOffset": 0
1938
+ }
1939
+ ],
1940
+ "md5sum": "0dfda91cfe8ba98fff5a402cfe1dc7df"
1941
+ },
1942
+ {
1943
+ "dataPath": "params_shard_44.bin",
1944
+ "format": "raw-shard",
1945
+ "nbytes": 25165824,
1946
+ "records": [
1947
+ {
1948
+ "name": "gpt_neox.layers.13.attention.query_key_value.weight",
1949
+ "shape": [
1950
+ 6144,
1951
+ 2048
1952
+ ],
1953
+ "dtype": "float16",
1954
+ "format": "f32-to-bf16",
1955
+ "nbytes": 25165824,
1956
+ "byteOffset": 0
1957
+ }
1958
+ ],
1959
+ "md5sum": "287808f7646050482dd9d4f73f3d71e1"
1960
+ },
1961
+ {
1962
+ "dataPath": "params_shard_45.bin",
1963
+ "format": "raw-shard",
1964
+ "nbytes": 33554432,
1965
+ "records": [
1966
+ {
1967
+ "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.weight",
1968
+ "shape": [
1969
+ 8192,
1970
+ 2048
1971
+ ],
1972
+ "dtype": "float16",
1973
+ "format": "f32-to-bf16",
1974
+ "nbytes": 33554432,
1975
+ "byteOffset": 0
1976
+ }
1977
+ ],
1978
+ "md5sum": "d556cb9c06f1ffaf060b18a650d7eec2"
1979
+ },
1980
+ {
1981
+ "dataPath": "params_shard_46.bin",
1982
+ "format": "raw-shard",
1983
+ "nbytes": 33554432,
1984
+ "records": [
1985
+ {
1986
+ "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.weight",
1987
+ "shape": [
1988
+ 2048,
1989
+ 8192
1990
+ ],
1991
+ "dtype": "float16",
1992
+ "format": "f32-to-bf16",
1993
+ "nbytes": 33554432,
1994
+ "byteOffset": 0
1995
+ }
1996
+ ],
1997
+ "md5sum": "c2e914507fcbf57fa0f6e2c0d0803550"
1998
+ },
1999
+ {
2000
+ "dataPath": "params_shard_47.bin",
2001
+ "format": "raw-shard",
2002
+ "nbytes": 25165824,
2003
+ "records": [
2004
+ {
2005
+ "name": "gpt_neox.layers.14.attention.query_key_value.weight",
2006
+ "shape": [
2007
+ 6144,
2008
+ 2048
2009
+ ],
2010
+ "dtype": "float16",
2011
+ "format": "f32-to-bf16",
2012
+ "nbytes": 25165824,
2013
+ "byteOffset": 0
2014
+ }
2015
+ ],
2016
+ "md5sum": "6dc1cb91a703d693796b3135b7a95ff6"
2017
+ },
2018
+ {
2019
+ "dataPath": "params_shard_48.bin",
2020
+ "format": "raw-shard",
2021
+ "nbytes": 33554432,
2022
+ "records": [
2023
+ {
2024
+ "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.weight",
2025
+ "shape": [
2026
+ 8192,
2027
+ 2048
2028
+ ],
2029
+ "dtype": "float16",
2030
+ "format": "f32-to-bf16",
2031
+ "nbytes": 33554432,
2032
+ "byteOffset": 0
2033
+ }
2034
+ ],
2035
+ "md5sum": "9905cbb2b08b5ffbc8b154005dd5ed0c"
2036
+ },
2037
+ {
2038
+ "dataPath": "params_shard_49.bin",
2039
+ "format": "raw-shard",
2040
+ "nbytes": 33554432,
2041
+ "records": [
2042
+ {
2043
+ "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.weight",
2044
+ "shape": [
2045
+ 2048,
2046
+ 8192
2047
+ ],
2048
+ "dtype": "float16",
2049
+ "format": "f32-to-bf16",
2050
+ "nbytes": 33554432,
2051
+ "byteOffset": 0
2052
+ }
2053
+ ],
2054
+ "md5sum": "62120c836ef22bd90e4391cd148ccd2f"
2055
+ },
2056
+ {
2057
+ "dataPath": "params_shard_50.bin",
2058
+ "format": "raw-shard",
2059
+ "nbytes": 25165824,
2060
+ "records": [
2061
+ {
2062
+ "name": "gpt_neox.layers.15.attention.query_key_value.weight",
2063
+ "shape": [
2064
+ 6144,
2065
+ 2048
2066
+ ],
2067
+ "dtype": "float16",
2068
+ "format": "f32-to-bf16",
2069
+ "nbytes": 25165824,
2070
+ "byteOffset": 0
2071
+ }
2072
+ ],
2073
+ "md5sum": "f3756f5a5824708fd51f3de6e54449e5"
2074
+ },
2075
+ {
2076
+ "dataPath": "params_shard_51.bin",
2077
+ "format": "raw-shard",
2078
+ "nbytes": 25325568,
2079
+ "records": [
2080
+ {
2081
+ "name": "gpt_neox.layers.12.attention.dense.weight",
2082
+ "shape": [
2083
+ 2048,
2084
+ 2048
2085
+ ],
2086
+ "dtype": "float16",
2087
+ "format": "f32-to-bf16",
2088
+ "nbytes": 8388608,
2089
+ "byteOffset": 0
2090
+ },
2091
+ {
2092
+ "name": "gpt_neox.layers.12.attention.dense.bias",
2093
+ "shape": [
2094
+ 2048
2095
+ ],
2096
+ "dtype": "float16",
2097
+ "format": "f32-to-bf16",
2098
+ "nbytes": 4096,
2099
+ "byteOffset": 8388608
2100
+ },
2101
+ {
2102
+ "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias",
2103
+ "shape": [
2104
+ 8192
2105
+ ],
2106
+ "dtype": "bfloat16",
2107
+ "format": "raw",
2108
+ "nbytes": 16384,
2109
+ "byteOffset": 8392704
2110
+ },
2111
+ {
2112
+ "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias",
2113
+ "shape": [
2114
+ 2048
2115
+ ],
2116
+ "dtype": "bfloat16",
2117
+ "format": "raw",
2118
+ "nbytes": 4096,
2119
+ "byteOffset": 8409088
2120
+ },
2121
+ {
2122
+ "name": "gpt_neox.layers.13.input_layernorm.weight",
2123
+ "shape": [
2124
+ 2048
2125
+ ],
2126
+ "dtype": "float16",
2127
+ "format": "f32-to-bf16",
2128
+ "nbytes": 4096,
2129
+ "byteOffset": 8413184
2130
+ },
2131
+ {
2132
+ "name": "gpt_neox.layers.13.input_layernorm.bias",
2133
+ "shape": [
2134
+ 2048
2135
+ ],
2136
+ "dtype": "float16",
2137
+ "format": "f32-to-bf16",
2138
+ "nbytes": 4096,
2139
+ "byteOffset": 8417280
2140
+ },
2141
+ {
2142
+ "name": "gpt_neox.layers.13.post_attention_layernorm.weight",
2143
+ "shape": [
2144
+ 2048
2145
+ ],
2146
+ "dtype": "float16",
2147
+ "format": "f32-to-bf16",
2148
+ "nbytes": 4096,
2149
+ "byteOffset": 8421376
2150
+ },
2151
+ {
2152
+ "name": "gpt_neox.layers.13.post_attention_layernorm.bias",
2153
+ "shape": [
2154
+ 2048
2155
+ ],
2156
+ "dtype": "float16",
2157
+ "format": "f32-to-bf16",
2158
+ "nbytes": 4096,
2159
+ "byteOffset": 8425472
2160
+ },
2161
+ {
2162
+ "name": "gpt_neox.layers.13.attention.query_key_value.bias",
2163
+ "shape": [
2164
+ 6144
2165
+ ],
2166
+ "dtype": "float16",
2167
+ "format": "f32-to-bf16",
2168
+ "nbytes": 12288,
2169
+ "byteOffset": 8429568
2170
+ },
2171
+ {
2172
+ "name": "gpt_neox.layers.13.attention.dense.weight",
2173
+ "shape": [
2174
+ 2048,
2175
+ 2048
2176
+ ],
2177
+ "dtype": "float16",
2178
+ "format": "f32-to-bf16",
2179
+ "nbytes": 8388608,
2180
+ "byteOffset": 8441856
2181
+ },
2182
+ {
2183
+ "name": "gpt_neox.layers.13.attention.dense.bias",
2184
+ "shape": [
2185
+ 2048
2186
+ ],
2187
+ "dtype": "float16",
2188
+ "format": "f32-to-bf16",
2189
+ "nbytes": 4096,
2190
+ "byteOffset": 16830464
2191
+ },
2192
+ {
2193
+ "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias",
2194
+ "shape": [
2195
+ 8192
2196
+ ],
2197
+ "dtype": "bfloat16",
2198
+ "format": "raw",
2199
+ "nbytes": 16384,
2200
+ "byteOffset": 16834560
2201
+ },
2202
+ {
2203
+ "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias",
2204
+ "shape": [
2205
+ 2048
2206
+ ],
2207
+ "dtype": "bfloat16",
2208
+ "format": "raw",
2209
+ "nbytes": 4096,
2210
+ "byteOffset": 16850944
2211
+ },
2212
+ {
2213
+ "name": "gpt_neox.layers.14.input_layernorm.weight",
2214
+ "shape": [
2215
+ 2048
2216
+ ],
2217
+ "dtype": "float16",
2218
+ "format": "f32-to-bf16",
2219
+ "nbytes": 4096,
2220
+ "byteOffset": 16855040
2221
+ },
2222
+ {
2223
+ "name": "gpt_neox.layers.14.input_layernorm.bias",
2224
+ "shape": [
2225
+ 2048
2226
+ ],
2227
+ "dtype": "float16",
2228
+ "format": "f32-to-bf16",
2229
+ "nbytes": 4096,
2230
+ "byteOffset": 16859136
2231
+ },
2232
+ {
2233
+ "name": "gpt_neox.layers.14.post_attention_layernorm.weight",
2234
+ "shape": [
2235
+ 2048
2236
+ ],
2237
+ "dtype": "float16",
2238
+ "format": "f32-to-bf16",
2239
+ "nbytes": 4096,
2240
+ "byteOffset": 16863232
2241
+ },
2242
+ {
2243
+ "name": "gpt_neox.layers.14.post_attention_layernorm.bias",
2244
+ "shape": [
2245
+ 2048
2246
+ ],
2247
+ "dtype": "float16",
2248
+ "format": "f32-to-bf16",
2249
+ "nbytes": 4096,
2250
+ "byteOffset": 16867328
2251
+ },
2252
+ {
2253
+ "name": "gpt_neox.layers.14.attention.query_key_value.bias",
2254
+ "shape": [
2255
+ 6144
2256
+ ],
2257
+ "dtype": "float16",
2258
+ "format": "f32-to-bf16",
2259
+ "nbytes": 12288,
2260
+ "byteOffset": 16871424
2261
+ },
2262
+ {
2263
+ "name": "gpt_neox.layers.14.attention.dense.weight",
2264
+ "shape": [
2265
+ 2048,
2266
+ 2048
2267
+ ],
2268
+ "dtype": "float16",
2269
+ "format": "f32-to-bf16",
2270
+ "nbytes": 8388608,
2271
+ "byteOffset": 16883712
2272
+ },
2273
+ {
2274
+ "name": "gpt_neox.layers.14.attention.dense.bias",
2275
+ "shape": [
2276
+ 2048
2277
+ ],
2278
+ "dtype": "float16",
2279
+ "format": "f32-to-bf16",
2280
+ "nbytes": 4096,
2281
+ "byteOffset": 25272320
2282
+ },
2283
+ {
2284
+ "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias",
2285
+ "shape": [
2286
+ 8192
2287
+ ],
2288
+ "dtype": "bfloat16",
2289
+ "format": "raw",
2290
+ "nbytes": 16384,
2291
+ "byteOffset": 25276416
2292
+ },
2293
+ {
2294
+ "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias",
2295
+ "shape": [
2296
+ 2048
2297
+ ],
2298
+ "dtype": "bfloat16",
2299
+ "format": "raw",
2300
+ "nbytes": 4096,
2301
+ "byteOffset": 25292800
2302
+ },
2303
+ {
2304
+ "name": "gpt_neox.layers.15.input_layernorm.weight",
2305
+ "shape": [
2306
+ 2048
2307
+ ],
2308
+ "dtype": "float16",
2309
+ "format": "f32-to-bf16",
2310
+ "nbytes": 4096,
2311
+ "byteOffset": 25296896
2312
+ },
2313
+ {
2314
+ "name": "gpt_neox.layers.15.input_layernorm.bias",
2315
+ "shape": [
2316
+ 2048
2317
+ ],
2318
+ "dtype": "float16",
2319
+ "format": "f32-to-bf16",
2320
+ "nbytes": 4096,
2321
+ "byteOffset": 25300992
2322
+ },
2323
+ {
2324
+ "name": "gpt_neox.layers.15.post_attention_layernorm.weight",
2325
+ "shape": [
2326
+ 2048
2327
+ ],
2328
+ "dtype": "float16",
2329
+ "format": "f32-to-bf16",
2330
+ "nbytes": 4096,
2331
+ "byteOffset": 25305088
2332
+ },
2333
+ {
2334
+ "name": "gpt_neox.layers.15.post_attention_layernorm.bias",
2335
+ "shape": [
2336
+ 2048
2337
+ ],
2338
+ "dtype": "float16",
2339
+ "format": "f32-to-bf16",
2340
+ "nbytes": 4096,
2341
+ "byteOffset": 25309184
2342
+ },
2343
+ {
2344
+ "name": "gpt_neox.layers.15.attention.query_key_value.bias",
2345
+ "shape": [
2346
+ 6144
2347
+ ],
2348
+ "dtype": "float16",
2349
+ "format": "f32-to-bf16",
2350
+ "nbytes": 12288,
2351
+ "byteOffset": 25313280
2352
+ }
2353
+ ],
2354
+ "md5sum": "b7ae6066328d6d58cdba7752718dea43"
2355
+ },
2356
+ {
2357
+ "dataPath": "params_shard_52.bin",
2358
+ "format": "raw-shard",
2359
+ "nbytes": 33554432,
2360
+ "records": [
2361
+ {
2362
+ "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.weight",
2363
+ "shape": [
2364
+ 8192,
2365
+ 2048
2366
+ ],
2367
+ "dtype": "float16",
2368
+ "format": "f32-to-bf16",
2369
+ "nbytes": 33554432,
2370
+ "byteOffset": 0
2371
+ }
2372
+ ],
2373
+ "md5sum": "362174f589aa7225e0ed1396c68b612b"
2374
+ },
2375
+ {
2376
+ "dataPath": "params_shard_53.bin",
2377
+ "format": "raw-shard",
2378
+ "nbytes": 33554432,
2379
+ "records": [
2380
+ {
2381
+ "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.weight",
2382
+ "shape": [
2383
+ 2048,
2384
+ 8192
2385
+ ],
2386
+ "dtype": "float16",
2387
+ "format": "f32-to-bf16",
2388
+ "nbytes": 33554432,
2389
+ "byteOffset": 0
2390
+ }
2391
+ ],
2392
+ "md5sum": "abb280cf7c20e138d18a8f9cf85322c8"
2393
+ },
2394
+ {
2395
+ "dataPath": "params_shard_54.bin",
2396
+ "format": "raw-shard",
2397
+ "nbytes": 206045184,
2398
+ "records": [
2399
+ {
2400
+ "name": "embed_out.weight",
2401
+ "shape": [
2402
+ 50304,
2403
+ 2048
2404
+ ],
2405
+ "dtype": "float16",
2406
+ "format": "f32-to-bf16",
2407
+ "nbytes": 206045184,
2408
+ "byteOffset": 0
2409
+ }
2410
+ ],
2411
+ "md5sum": "3939e915eb3bd60919c8b6d517ea99f6"
2412
+ },
2413
+ {
2414
+ "dataPath": "params_shard_55.bin",
2415
+ "format": "raw-shard",
2416
+ "nbytes": 8421376,
2417
+ "records": [
2418
+ {
2419
+ "name": "gpt_neox.layers.15.attention.dense.weight",
2420
+ "shape": [
2421
+ 2048,
2422
+ 2048
2423
+ ],
2424
+ "dtype": "float16",
2425
+ "format": "f32-to-bf16",
2426
+ "nbytes": 8388608,
2427
+ "byteOffset": 0
2428
+ },
2429
+ {
2430
+ "name": "gpt_neox.layers.15.attention.dense.bias",
2431
+ "shape": [
2432
+ 2048
2433
+ ],
2434
+ "dtype": "float16",
2435
+ "format": "f32-to-bf16",
2436
+ "nbytes": 4096,
2437
+ "byteOffset": 8388608
2438
+ },
2439
+ {
2440
+ "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias",
2441
+ "shape": [
2442
+ 8192
2443
+ ],
2444
+ "dtype": "bfloat16",
2445
+ "format": "raw",
2446
+ "nbytes": 16384,
2447
+ "byteOffset": 8392704
2448
+ },
2449
+ {
2450
+ "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias",
2451
+ "shape": [
2452
+ 2048
2453
+ ],
2454
+ "dtype": "bfloat16",
2455
+ "format": "raw",
2456
+ "nbytes": 4096,
2457
+ "byteOffset": 8409088
2458
+ },
2459
+ {
2460
+ "name": "gpt_neox.final_layer_norm.weight",
2461
+ "shape": [
2462
+ 2048
2463
+ ],
2464
+ "dtype": "float16",
2465
+ "format": "f32-to-bf16",
2466
+ "nbytes": 4096,
2467
+ "byteOffset": 8413184
2468
+ },
2469
+ {
2470
+ "name": "gpt_neox.final_layer_norm.bias",
2471
+ "shape": [
2472
+ 2048
2473
+ ],
2474
+ "dtype": "float16",
2475
+ "format": "f32-to-bf16",
2476
+ "nbytes": 4096,
2477
+ "byteOffset": 8417280
2478
+ }
2479
+ ],
2480
+ "md5sum": "4eaa42565d775e54b8a02497b2a7e15d"
2481
+ }
2482
+ ]
2483
+ }
ndarray-cache.json ADDED
@@ -0,0 +1,2483 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "ParamSize": 196,
4
+ "ParamBytes": 2023890944.0,
5
+ "BitsPerParam": 16.002590914795338
6
+ },
7
+ "records": [
8
+ {
9
+ "dataPath": "params_shard_0.bin",
10
+ "format": "raw-shard",
11
+ "nbytes": 206045184,
12
+ "records": [
13
+ {
14
+ "name": "gpt_neox.embed_in.weight",
15
+ "shape": [
16
+ 50304,
17
+ 2048
18
+ ],
19
+ "dtype": "float16",
20
+ "format": "f32-to-bf16",
21
+ "nbytes": 206045184,
22
+ "byteOffset": 0
23
+ }
24
+ ],
25
+ "md5sum": "da22c1a38504f2cdb102f78567ef6329"
26
+ },
27
+ {
28
+ "dataPath": "params_shard_1.bin",
29
+ "format": "raw-shard",
30
+ "nbytes": 25194496,
31
+ "records": [
32
+ {
33
+ "name": "gpt_neox.layers.0.input_layernorm.weight",
34
+ "shape": [
35
+ 2048
36
+ ],
37
+ "dtype": "float16",
38
+ "format": "f32-to-bf16",
39
+ "nbytes": 4096,
40
+ "byteOffset": 0
41
+ },
42
+ {
43
+ "name": "gpt_neox.layers.0.input_layernorm.bias",
44
+ "shape": [
45
+ 2048
46
+ ],
47
+ "dtype": "float16",
48
+ "format": "f32-to-bf16",
49
+ "nbytes": 4096,
50
+ "byteOffset": 4096
51
+ },
52
+ {
53
+ "name": "gpt_neox.layers.0.post_attention_layernorm.weight",
54
+ "shape": [
55
+ 2048
56
+ ],
57
+ "dtype": "float16",
58
+ "format": "f32-to-bf16",
59
+ "nbytes": 4096,
60
+ "byteOffset": 8192
61
+ },
62
+ {
63
+ "name": "gpt_neox.layers.0.post_attention_layernorm.bias",
64
+ "shape": [
65
+ 2048
66
+ ],
67
+ "dtype": "float16",
68
+ "format": "f32-to-bf16",
69
+ "nbytes": 4096,
70
+ "byteOffset": 12288
71
+ },
72
+ {
73
+ "name": "gpt_neox.layers.0.attention.query_key_value.weight",
74
+ "shape": [
75
+ 6144,
76
+ 2048
77
+ ],
78
+ "dtype": "float16",
79
+ "format": "f32-to-bf16",
80
+ "nbytes": 25165824,
81
+ "byteOffset": 16384
82
+ },
83
+ {
84
+ "name": "gpt_neox.layers.0.attention.query_key_value.bias",
85
+ "shape": [
86
+ 6144
87
+ ],
88
+ "dtype": "float16",
89
+ "format": "f32-to-bf16",
90
+ "nbytes": 12288,
91
+ "byteOffset": 25182208
92
+ }
93
+ ],
94
+ "md5sum": "e2d7cc2fa350f011ed0dbee17a7ac7e8"
95
+ },
96
+ {
97
+ "dataPath": "params_shard_2.bin",
98
+ "format": "raw-shard",
99
+ "nbytes": 33554432,
100
+ "records": [
101
+ {
102
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight",
103
+ "shape": [
104
+ 8192,
105
+ 2048
106
+ ],
107
+ "dtype": "float16",
108
+ "format": "f32-to-bf16",
109
+ "nbytes": 33554432,
110
+ "byteOffset": 0
111
+ }
112
+ ],
113
+ "md5sum": "e4a3272a25b09a7f71da4953d754e374"
114
+ },
115
+ {
116
+ "dataPath": "params_shard_3.bin",
117
+ "format": "raw-shard",
118
+ "nbytes": 33554432,
119
+ "records": [
120
+ {
121
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight",
122
+ "shape": [
123
+ 2048,
124
+ 8192
125
+ ],
126
+ "dtype": "float16",
127
+ "format": "f32-to-bf16",
128
+ "nbytes": 33554432,
129
+ "byteOffset": 0
130
+ }
131
+ ],
132
+ "md5sum": "e458ab97e79a42f7d337f26ed979dafa"
133
+ },
134
+ {
135
+ "dataPath": "params_shard_4.bin",
136
+ "format": "raw-shard",
137
+ "nbytes": 25165824,
138
+ "records": [
139
+ {
140
+ "name": "gpt_neox.layers.1.attention.query_key_value.weight",
141
+ "shape": [
142
+ 6144,
143
+ 2048
144
+ ],
145
+ "dtype": "float16",
146
+ "format": "f32-to-bf16",
147
+ "nbytes": 25165824,
148
+ "byteOffset": 0
149
+ }
150
+ ],
151
+ "md5sum": "47dd12a2f3b316282581e8d851f16361"
152
+ },
153
+ {
154
+ "dataPath": "params_shard_5.bin",
155
+ "format": "raw-shard",
156
+ "nbytes": 33554432,
157
+ "records": [
158
+ {
159
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight",
160
+ "shape": [
161
+ 8192,
162
+ 2048
163
+ ],
164
+ "dtype": "float16",
165
+ "format": "f32-to-bf16",
166
+ "nbytes": 33554432,
167
+ "byteOffset": 0
168
+ }
169
+ ],
170
+ "md5sum": "8b376542ee52be1a671247bc19759893"
171
+ },
172
+ {
173
+ "dataPath": "params_shard_6.bin",
174
+ "format": "raw-shard",
175
+ "nbytes": 33554432,
176
+ "records": [
177
+ {
178
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight",
179
+ "shape": [
180
+ 2048,
181
+ 8192
182
+ ],
183
+ "dtype": "float16",
184
+ "format": "f32-to-bf16",
185
+ "nbytes": 33554432,
186
+ "byteOffset": 0
187
+ }
188
+ ],
189
+ "md5sum": "8d429e502c7ffe5efcdeff2840694943"
190
+ },
191
+ {
192
+ "dataPath": "params_shard_7.bin",
193
+ "format": "raw-shard",
194
+ "nbytes": 25165824,
195
+ "records": [
196
+ {
197
+ "name": "gpt_neox.layers.2.attention.query_key_value.weight",
198
+ "shape": [
199
+ 6144,
200
+ 2048
201
+ ],
202
+ "dtype": "float16",
203
+ "format": "f32-to-bf16",
204
+ "nbytes": 25165824,
205
+ "byteOffset": 0
206
+ }
207
+ ],
208
+ "md5sum": "f6b6c9c4ce289692d543d75a0ba5693c"
209
+ },
210
+ {
211
+ "dataPath": "params_shard_8.bin",
212
+ "format": "raw-shard",
213
+ "nbytes": 33554432,
214
+ "records": [
215
+ {
216
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight",
217
+ "shape": [
218
+ 8192,
219
+ 2048
220
+ ],
221
+ "dtype": "float16",
222
+ "format": "f32-to-bf16",
223
+ "nbytes": 33554432,
224
+ "byteOffset": 0
225
+ }
226
+ ],
227
+ "md5sum": "267dc9558c1f41aef67d0e1f022be352"
228
+ },
229
+ {
230
+ "dataPath": "params_shard_9.bin",
231
+ "format": "raw-shard",
232
+ "nbytes": 33554432,
233
+ "records": [
234
+ {
235
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight",
236
+ "shape": [
237
+ 2048,
238
+ 8192
239
+ ],
240
+ "dtype": "float16",
241
+ "format": "f32-to-bf16",
242
+ "nbytes": 33554432,
243
+ "byteOffset": 0
244
+ }
245
+ ],
246
+ "md5sum": "cee8c73decb5ea89ffbef7ee86bf8689"
247
+ },
248
+ {
249
+ "dataPath": "params_shard_10.bin",
250
+ "format": "raw-shard",
251
+ "nbytes": 25165824,
252
+ "records": [
253
+ {
254
+ "name": "gpt_neox.layers.3.attention.query_key_value.weight",
255
+ "shape": [
256
+ 6144,
257
+ 2048
258
+ ],
259
+ "dtype": "float16",
260
+ "format": "f32-to-bf16",
261
+ "nbytes": 25165824,
262
+ "byteOffset": 0
263
+ }
264
+ ],
265
+ "md5sum": "db045065ae5ef84f1464bc836113ef76"
266
+ },
267
+ {
268
+ "dataPath": "params_shard_11.bin",
269
+ "format": "raw-shard",
270
+ "nbytes": 25325568,
271
+ "records": [
272
+ {
273
+ "name": "gpt_neox.layers.0.attention.dense.weight",
274
+ "shape": [
275
+ 2048,
276
+ 2048
277
+ ],
278
+ "dtype": "float16",
279
+ "format": "f32-to-bf16",
280
+ "nbytes": 8388608,
281
+ "byteOffset": 0
282
+ },
283
+ {
284
+ "name": "gpt_neox.layers.0.attention.dense.bias",
285
+ "shape": [
286
+ 2048
287
+ ],
288
+ "dtype": "float16",
289
+ "format": "f32-to-bf16",
290
+ "nbytes": 4096,
291
+ "byteOffset": 8388608
292
+ },
293
+ {
294
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
295
+ "shape": [
296
+ 8192
297
+ ],
298
+ "dtype": "float32",
299
+ "format": "f32-to-bf16",
300
+ "nbytes": 16384,
301
+ "byteOffset": 8392704
302
+ },
303
+ {
304
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
305
+ "shape": [
306
+ 2048
307
+ ],
308
+ "dtype": "float32",
309
+ "format": "f32-to-bf16",
310
+ "nbytes": 4096,
311
+ "byteOffset": 8409088
312
+ },
313
+ {
314
+ "name": "gpt_neox.layers.1.input_layernorm.weight",
315
+ "shape": [
316
+ 2048
317
+ ],
318
+ "dtype": "float16",
319
+ "format": "f32-to-bf16",
320
+ "nbytes": 4096,
321
+ "byteOffset": 8413184
322
+ },
323
+ {
324
+ "name": "gpt_neox.layers.1.input_layernorm.bias",
325
+ "shape": [
326
+ 2048
327
+ ],
328
+ "dtype": "float16",
329
+ "format": "f32-to-bf16",
330
+ "nbytes": 4096,
331
+ "byteOffset": 8417280
332
+ },
333
+ {
334
+ "name": "gpt_neox.layers.1.post_attention_layernorm.weight",
335
+ "shape": [
336
+ 2048
337
+ ],
338
+ "dtype": "float16",
339
+ "format": "f32-to-bf16",
340
+ "nbytes": 4096,
341
+ "byteOffset": 8421376
342
+ },
343
+ {
344
+ "name": "gpt_neox.layers.1.post_attention_layernorm.bias",
345
+ "shape": [
346
+ 2048
347
+ ],
348
+ "dtype": "float16",
349
+ "format": "f32-to-bf16",
350
+ "nbytes": 4096,
351
+ "byteOffset": 8425472
352
+ },
353
+ {
354
+ "name": "gpt_neox.layers.1.attention.query_key_value.bias",
355
+ "shape": [
356
+ 6144
357
+ ],
358
+ "dtype": "float16",
359
+ "format": "f32-to-bf16",
360
+ "nbytes": 12288,
361
+ "byteOffset": 8429568
362
+ },
363
+ {
364
+ "name": "gpt_neox.layers.1.attention.dense.weight",
365
+ "shape": [
366
+ 2048,
367
+ 2048
368
+ ],
369
+ "dtype": "float16",
370
+ "format": "f32-to-bf16",
371
+ "nbytes": 8388608,
372
+ "byteOffset": 8441856
373
+ },
374
+ {
375
+ "name": "gpt_neox.layers.1.attention.dense.bias",
376
+ "shape": [
377
+ 2048
378
+ ],
379
+ "dtype": "float16",
380
+ "format": "f32-to-bf16",
381
+ "nbytes": 4096,
382
+ "byteOffset": 16830464
383
+ },
384
+ {
385
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
386
+ "shape": [
387
+ 8192
388
+ ],
389
+ "dtype": "float32",
390
+ "format": "f32-to-bf16",
391
+ "nbytes": 16384,
392
+ "byteOffset": 16834560
393
+ },
394
+ {
395
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
396
+ "shape": [
397
+ 2048
398
+ ],
399
+ "dtype": "float32",
400
+ "format": "f32-to-bf16",
401
+ "nbytes": 4096,
402
+ "byteOffset": 16850944
403
+ },
404
+ {
405
+ "name": "gpt_neox.layers.2.input_layernorm.weight",
406
+ "shape": [
407
+ 2048
408
+ ],
409
+ "dtype": "float16",
410
+ "format": "f32-to-bf16",
411
+ "nbytes": 4096,
412
+ "byteOffset": 16855040
413
+ },
414
+ {
415
+ "name": "gpt_neox.layers.2.input_layernorm.bias",
416
+ "shape": [
417
+ 2048
418
+ ],
419
+ "dtype": "float16",
420
+ "format": "f32-to-bf16",
421
+ "nbytes": 4096,
422
+ "byteOffset": 16859136
423
+ },
424
+ {
425
+ "name": "gpt_neox.layers.2.post_attention_layernorm.weight",
426
+ "shape": [
427
+ 2048
428
+ ],
429
+ "dtype": "float16",
430
+ "format": "f32-to-bf16",
431
+ "nbytes": 4096,
432
+ "byteOffset": 16863232
433
+ },
434
+ {
435
+ "name": "gpt_neox.layers.2.post_attention_layernorm.bias",
436
+ "shape": [
437
+ 2048
438
+ ],
439
+ "dtype": "float16",
440
+ "format": "f32-to-bf16",
441
+ "nbytes": 4096,
442
+ "byteOffset": 16867328
443
+ },
444
+ {
445
+ "name": "gpt_neox.layers.2.attention.query_key_value.bias",
446
+ "shape": [
447
+ 6144
448
+ ],
449
+ "dtype": "float16",
450
+ "format": "f32-to-bf16",
451
+ "nbytes": 12288,
452
+ "byteOffset": 16871424
453
+ },
454
+ {
455
+ "name": "gpt_neox.layers.2.attention.dense.weight",
456
+ "shape": [
457
+ 2048,
458
+ 2048
459
+ ],
460
+ "dtype": "float16",
461
+ "format": "f32-to-bf16",
462
+ "nbytes": 8388608,
463
+ "byteOffset": 16883712
464
+ },
465
+ {
466
+ "name": "gpt_neox.layers.2.attention.dense.bias",
467
+ "shape": [
468
+ 2048
469
+ ],
470
+ "dtype": "float16",
471
+ "format": "f32-to-bf16",
472
+ "nbytes": 4096,
473
+ "byteOffset": 25272320
474
+ },
475
+ {
476
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
477
+ "shape": [
478
+ 8192
479
+ ],
480
+ "dtype": "float32",
481
+ "format": "f32-to-bf16",
482
+ "nbytes": 16384,
483
+ "byteOffset": 25276416
484
+ },
485
+ {
486
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
487
+ "shape": [
488
+ 2048
489
+ ],
490
+ "dtype": "float32",
491
+ "format": "f32-to-bf16",
492
+ "nbytes": 4096,
493
+ "byteOffset": 25292800
494
+ },
495
+ {
496
+ "name": "gpt_neox.layers.3.input_layernorm.weight",
497
+ "shape": [
498
+ 2048
499
+ ],
500
+ "dtype": "float16",
501
+ "format": "f32-to-bf16",
502
+ "nbytes": 4096,
503
+ "byteOffset": 25296896
504
+ },
505
+ {
506
+ "name": "gpt_neox.layers.3.input_layernorm.bias",
507
+ "shape": [
508
+ 2048
509
+ ],
510
+ "dtype": "float16",
511
+ "format": "f32-to-bf16",
512
+ "nbytes": 4096,
513
+ "byteOffset": 25300992
514
+ },
515
+ {
516
+ "name": "gpt_neox.layers.3.post_attention_layernorm.weight",
517
+ "shape": [
518
+ 2048
519
+ ],
520
+ "dtype": "float16",
521
+ "format": "f32-to-bf16",
522
+ "nbytes": 4096,
523
+ "byteOffset": 25305088
524
+ },
525
+ {
526
+ "name": "gpt_neox.layers.3.post_attention_layernorm.bias",
527
+ "shape": [
528
+ 2048
529
+ ],
530
+ "dtype": "float16",
531
+ "format": "f32-to-bf16",
532
+ "nbytes": 4096,
533
+ "byteOffset": 25309184
534
+ },
535
+ {
536
+ "name": "gpt_neox.layers.3.attention.query_key_value.bias",
537
+ "shape": [
538
+ 6144
539
+ ],
540
+ "dtype": "float16",
541
+ "format": "f32-to-bf16",
542
+ "nbytes": 12288,
543
+ "byteOffset": 25313280
544
+ }
545
+ ],
546
+ "md5sum": "a300f069dd9a566a4d85894be26bd41d"
547
+ },
548
+ {
549
+ "dataPath": "params_shard_12.bin",
550
+ "format": "raw-shard",
551
+ "nbytes": 33554432,
552
+ "records": [
553
+ {
554
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight",
555
+ "shape": [
556
+ 8192,
557
+ 2048
558
+ ],
559
+ "dtype": "float16",
560
+ "format": "f32-to-bf16",
561
+ "nbytes": 33554432,
562
+ "byteOffset": 0
563
+ }
564
+ ],
565
+ "md5sum": "3bf00c3ee5d95ad98f32ea93b3b6ca63"
566
+ },
567
+ {
568
+ "dataPath": "params_shard_13.bin",
569
+ "format": "raw-shard",
570
+ "nbytes": 33554432,
571
+ "records": [
572
+ {
573
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight",
574
+ "shape": [
575
+ 2048,
576
+ 8192
577
+ ],
578
+ "dtype": "float16",
579
+ "format": "f32-to-bf16",
580
+ "nbytes": 33554432,
581
+ "byteOffset": 0
582
+ }
583
+ ],
584
+ "md5sum": "2bc23e26ce74f84d28e0c5f69bf408b3"
585
+ },
586
+ {
587
+ "dataPath": "params_shard_14.bin",
588
+ "format": "raw-shard",
589
+ "nbytes": 25165824,
590
+ "records": [
591
+ {
592
+ "name": "gpt_neox.layers.4.attention.query_key_value.weight",
593
+ "shape": [
594
+ 6144,
595
+ 2048
596
+ ],
597
+ "dtype": "float16",
598
+ "format": "f32-to-bf16",
599
+ "nbytes": 25165824,
600
+ "byteOffset": 0
601
+ }
602
+ ],
603
+ "md5sum": "1aa107f47a4ca56262e01cec478fb5ed"
604
+ },
605
+ {
606
+ "dataPath": "params_shard_15.bin",
607
+ "format": "raw-shard",
608
+ "nbytes": 33554432,
609
+ "records": [
610
+ {
611
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight",
612
+ "shape": [
613
+ 8192,
614
+ 2048
615
+ ],
616
+ "dtype": "float16",
617
+ "format": "f32-to-bf16",
618
+ "nbytes": 33554432,
619
+ "byteOffset": 0
620
+ }
621
+ ],
622
+ "md5sum": "3c65c52252e8eb6acb5856d86e3bc051"
623
+ },
624
+ {
625
+ "dataPath": "params_shard_16.bin",
626
+ "format": "raw-shard",
627
+ "nbytes": 33554432,
628
+ "records": [
629
+ {
630
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight",
631
+ "shape": [
632
+ 2048,
633
+ 8192
634
+ ],
635
+ "dtype": "float16",
636
+ "format": "f32-to-bf16",
637
+ "nbytes": 33554432,
638
+ "byteOffset": 0
639
+ }
640
+ ],
641
+ "md5sum": "311a1252100864f529064f11e902487f"
642
+ },
643
+ {
644
+ "dataPath": "params_shard_17.bin",
645
+ "format": "raw-shard",
646
+ "nbytes": 25165824,
647
+ "records": [
648
+ {
649
+ "name": "gpt_neox.layers.5.attention.query_key_value.weight",
650
+ "shape": [
651
+ 6144,
652
+ 2048
653
+ ],
654
+ "dtype": "float16",
655
+ "format": "f32-to-bf16",
656
+ "nbytes": 25165824,
657
+ "byteOffset": 0
658
+ }
659
+ ],
660
+ "md5sum": "011f5a5c7cb3f338e981f263ed52f81b"
661
+ },
662
+ {
663
+ "dataPath": "params_shard_18.bin",
664
+ "format": "raw-shard",
665
+ "nbytes": 33554432,
666
+ "records": [
667
+ {
668
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight",
669
+ "shape": [
670
+ 8192,
671
+ 2048
672
+ ],
673
+ "dtype": "float16",
674
+ "format": "f32-to-bf16",
675
+ "nbytes": 33554432,
676
+ "byteOffset": 0
677
+ }
678
+ ],
679
+ "md5sum": "df4abbad1146d3fc204e354532df4c32"
680
+ },
681
+ {
682
+ "dataPath": "params_shard_19.bin",
683
+ "format": "raw-shard",
684
+ "nbytes": 33554432,
685
+ "records": [
686
+ {
687
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight",
688
+ "shape": [
689
+ 2048,
690
+ 8192
691
+ ],
692
+ "dtype": "float16",
693
+ "format": "f32-to-bf16",
694
+ "nbytes": 33554432,
695
+ "byteOffset": 0
696
+ }
697
+ ],
698
+ "md5sum": "553c08ac1659b464bf8dba3f262303a1"
699
+ },
700
+ {
701
+ "dataPath": "params_shard_20.bin",
702
+ "format": "raw-shard",
703
+ "nbytes": 25165824,
704
+ "records": [
705
+ {
706
+ "name": "gpt_neox.layers.6.attention.query_key_value.weight",
707
+ "shape": [
708
+ 6144,
709
+ 2048
710
+ ],
711
+ "dtype": "float16",
712
+ "format": "f32-to-bf16",
713
+ "nbytes": 25165824,
714
+ "byteOffset": 0
715
+ }
716
+ ],
717
+ "md5sum": "ba5ea4be88cb85fc9109dac76d309c05"
718
+ },
719
+ {
720
+ "dataPath": "params_shard_21.bin",
721
+ "format": "raw-shard",
722
+ "nbytes": 25325568,
723
+ "records": [
724
+ {
725
+ "name": "gpt_neox.layers.3.attention.dense.weight",
726
+ "shape": [
727
+ 2048,
728
+ 2048
729
+ ],
730
+ "dtype": "float16",
731
+ "format": "f32-to-bf16",
732
+ "nbytes": 8388608,
733
+ "byteOffset": 0
734
+ },
735
+ {
736
+ "name": "gpt_neox.layers.3.attention.dense.bias",
737
+ "shape": [
738
+ 2048
739
+ ],
740
+ "dtype": "float16",
741
+ "format": "f32-to-bf16",
742
+ "nbytes": 4096,
743
+ "byteOffset": 8388608
744
+ },
745
+ {
746
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
747
+ "shape": [
748
+ 8192
749
+ ],
750
+ "dtype": "float32",
751
+ "format": "f32-to-bf16",
752
+ "nbytes": 16384,
753
+ "byteOffset": 8392704
754
+ },
755
+ {
756
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
757
+ "shape": [
758
+ 2048
759
+ ],
760
+ "dtype": "float32",
761
+ "format": "f32-to-bf16",
762
+ "nbytes": 4096,
763
+ "byteOffset": 8409088
764
+ },
765
+ {
766
+ "name": "gpt_neox.layers.4.input_layernorm.weight",
767
+ "shape": [
768
+ 2048
769
+ ],
770
+ "dtype": "float16",
771
+ "format": "f32-to-bf16",
772
+ "nbytes": 4096,
773
+ "byteOffset": 8413184
774
+ },
775
+ {
776
+ "name": "gpt_neox.layers.4.input_layernorm.bias",
777
+ "shape": [
778
+ 2048
779
+ ],
780
+ "dtype": "float16",
781
+ "format": "f32-to-bf16",
782
+ "nbytes": 4096,
783
+ "byteOffset": 8417280
784
+ },
785
+ {
786
+ "name": "gpt_neox.layers.4.post_attention_layernorm.weight",
787
+ "shape": [
788
+ 2048
789
+ ],
790
+ "dtype": "float16",
791
+ "format": "f32-to-bf16",
792
+ "nbytes": 4096,
793
+ "byteOffset": 8421376
794
+ },
795
+ {
796
+ "name": "gpt_neox.layers.4.post_attention_layernorm.bias",
797
+ "shape": [
798
+ 2048
799
+ ],
800
+ "dtype": "float16",
801
+ "format": "f32-to-bf16",
802
+ "nbytes": 4096,
803
+ "byteOffset": 8425472
804
+ },
805
+ {
806
+ "name": "gpt_neox.layers.4.attention.query_key_value.bias",
807
+ "shape": [
808
+ 6144
809
+ ],
810
+ "dtype": "float16",
811
+ "format": "f32-to-bf16",
812
+ "nbytes": 12288,
813
+ "byteOffset": 8429568
814
+ },
815
+ {
816
+ "name": "gpt_neox.layers.4.attention.dense.weight",
817
+ "shape": [
818
+ 2048,
819
+ 2048
820
+ ],
821
+ "dtype": "float16",
822
+ "format": "f32-to-bf16",
823
+ "nbytes": 8388608,
824
+ "byteOffset": 8441856
825
+ },
826
+ {
827
+ "name": "gpt_neox.layers.4.attention.dense.bias",
828
+ "shape": [
829
+ 2048
830
+ ],
831
+ "dtype": "float16",
832
+ "format": "f32-to-bf16",
833
+ "nbytes": 4096,
834
+ "byteOffset": 16830464
835
+ },
836
+ {
837
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
838
+ "shape": [
839
+ 8192
840
+ ],
841
+ "dtype": "float32",
842
+ "format": "f32-to-bf16",
843
+ "nbytes": 16384,
844
+ "byteOffset": 16834560
845
+ },
846
+ {
847
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
848
+ "shape": [
849
+ 2048
850
+ ],
851
+ "dtype": "float32",
852
+ "format": "f32-to-bf16",
853
+ "nbytes": 4096,
854
+ "byteOffset": 16850944
855
+ },
856
+ {
857
+ "name": "gpt_neox.layers.5.input_layernorm.weight",
858
+ "shape": [
859
+ 2048
860
+ ],
861
+ "dtype": "float16",
862
+ "format": "f32-to-bf16",
863
+ "nbytes": 4096,
864
+ "byteOffset": 16855040
865
+ },
866
+ {
867
+ "name": "gpt_neox.layers.5.input_layernorm.bias",
868
+ "shape": [
869
+ 2048
870
+ ],
871
+ "dtype": "float16",
872
+ "format": "f32-to-bf16",
873
+ "nbytes": 4096,
874
+ "byteOffset": 16859136
875
+ },
876
+ {
877
+ "name": "gpt_neox.layers.5.post_attention_layernorm.weight",
878
+ "shape": [
879
+ 2048
880
+ ],
881
+ "dtype": "float16",
882
+ "format": "f32-to-bf16",
883
+ "nbytes": 4096,
884
+ "byteOffset": 16863232
885
+ },
886
+ {
887
+ "name": "gpt_neox.layers.5.post_attention_layernorm.bias",
888
+ "shape": [
889
+ 2048
890
+ ],
891
+ "dtype": "float16",
892
+ "format": "f32-to-bf16",
893
+ "nbytes": 4096,
894
+ "byteOffset": 16867328
895
+ },
896
+ {
897
+ "name": "gpt_neox.layers.5.attention.query_key_value.bias",
898
+ "shape": [
899
+ 6144
900
+ ],
901
+ "dtype": "float16",
902
+ "format": "f32-to-bf16",
903
+ "nbytes": 12288,
904
+ "byteOffset": 16871424
905
+ },
906
+ {
907
+ "name": "gpt_neox.layers.5.attention.dense.weight",
908
+ "shape": [
909
+ 2048,
910
+ 2048
911
+ ],
912
+ "dtype": "float16",
913
+ "format": "f32-to-bf16",
914
+ "nbytes": 8388608,
915
+ "byteOffset": 16883712
916
+ },
917
+ {
918
+ "name": "gpt_neox.layers.5.attention.dense.bias",
919
+ "shape": [
920
+ 2048
921
+ ],
922
+ "dtype": "float16",
923
+ "format": "f32-to-bf16",
924
+ "nbytes": 4096,
925
+ "byteOffset": 25272320
926
+ },
927
+ {
928
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
929
+ "shape": [
930
+ 8192
931
+ ],
932
+ "dtype": "float32",
933
+ "format": "f32-to-bf16",
934
+ "nbytes": 16384,
935
+ "byteOffset": 25276416
936
+ },
937
+ {
938
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
939
+ "shape": [
940
+ 2048
941
+ ],
942
+ "dtype": "float32",
943
+ "format": "f32-to-bf16",
944
+ "nbytes": 4096,
945
+ "byteOffset": 25292800
946
+ },
947
+ {
948
+ "name": "gpt_neox.layers.6.input_layernorm.weight",
949
+ "shape": [
950
+ 2048
951
+ ],
952
+ "dtype": "float16",
953
+ "format": "f32-to-bf16",
954
+ "nbytes": 4096,
955
+ "byteOffset": 25296896
956
+ },
957
+ {
958
+ "name": "gpt_neox.layers.6.input_layernorm.bias",
959
+ "shape": [
960
+ 2048
961
+ ],
962
+ "dtype": "float16",
963
+ "format": "f32-to-bf16",
964
+ "nbytes": 4096,
965
+ "byteOffset": 25300992
966
+ },
967
+ {
968
+ "name": "gpt_neox.layers.6.post_attention_layernorm.weight",
969
+ "shape": [
970
+ 2048
971
+ ],
972
+ "dtype": "float16",
973
+ "format": "f32-to-bf16",
974
+ "nbytes": 4096,
975
+ "byteOffset": 25305088
976
+ },
977
+ {
978
+ "name": "gpt_neox.layers.6.post_attention_layernorm.bias",
979
+ "shape": [
980
+ 2048
981
+ ],
982
+ "dtype": "float16",
983
+ "format": "f32-to-bf16",
984
+ "nbytes": 4096,
985
+ "byteOffset": 25309184
986
+ },
987
+ {
988
+ "name": "gpt_neox.layers.6.attention.query_key_value.bias",
989
+ "shape": [
990
+ 6144
991
+ ],
992
+ "dtype": "float16",
993
+ "format": "f32-to-bf16",
994
+ "nbytes": 12288,
995
+ "byteOffset": 25313280
996
+ }
997
+ ],
998
+ "md5sum": "0f848458ed7ba85e561c6ac6953346aa"
999
+ },
1000
+ {
1001
+ "dataPath": "params_shard_22.bin",
1002
+ "format": "raw-shard",
1003
+ "nbytes": 33554432,
1004
+ "records": [
1005
+ {
1006
+ "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.weight",
1007
+ "shape": [
1008
+ 8192,
1009
+ 2048
1010
+ ],
1011
+ "dtype": "float16",
1012
+ "format": "f32-to-bf16",
1013
+ "nbytes": 33554432,
1014
+ "byteOffset": 0
1015
+ }
1016
+ ],
1017
+ "md5sum": "c483f21a9a023e8537dcb3a97635ac32"
1018
+ },
1019
+ {
1020
+ "dataPath": "params_shard_23.bin",
1021
+ "format": "raw-shard",
1022
+ "nbytes": 33554432,
1023
+ "records": [
1024
+ {
1025
+ "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.weight",
1026
+ "shape": [
1027
+ 2048,
1028
+ 8192
1029
+ ],
1030
+ "dtype": "float16",
1031
+ "format": "f32-to-bf16",
1032
+ "nbytes": 33554432,
1033
+ "byteOffset": 0
1034
+ }
1035
+ ],
1036
+ "md5sum": "1b8f8a2187e8fe0821a62f6aecc43903"
1037
+ },
1038
+ {
1039
+ "dataPath": "params_shard_24.bin",
1040
+ "format": "raw-shard",
1041
+ "nbytes": 25165824,
1042
+ "records": [
1043
+ {
1044
+ "name": "gpt_neox.layers.7.attention.query_key_value.weight",
1045
+ "shape": [
1046
+ 6144,
1047
+ 2048
1048
+ ],
1049
+ "dtype": "float16",
1050
+ "format": "f32-to-bf16",
1051
+ "nbytes": 25165824,
1052
+ "byteOffset": 0
1053
+ }
1054
+ ],
1055
+ "md5sum": "0c3a014c3c16ecf900cabc943c57a8b4"
1056
+ },
1057
+ {
1058
+ "dataPath": "params_shard_25.bin",
1059
+ "format": "raw-shard",
1060
+ "nbytes": 33554432,
1061
+ "records": [
1062
+ {
1063
+ "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.weight",
1064
+ "shape": [
1065
+ 8192,
1066
+ 2048
1067
+ ],
1068
+ "dtype": "float16",
1069
+ "format": "f32-to-bf16",
1070
+ "nbytes": 33554432,
1071
+ "byteOffset": 0
1072
+ }
1073
+ ],
1074
+ "md5sum": "3a068195dd649252ba4aab5ebff92b9c"
1075
+ },
1076
+ {
1077
+ "dataPath": "params_shard_26.bin",
1078
+ "format": "raw-shard",
1079
+ "nbytes": 33554432,
1080
+ "records": [
1081
+ {
1082
+ "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.weight",
1083
+ "shape": [
1084
+ 2048,
1085
+ 8192
1086
+ ],
1087
+ "dtype": "float16",
1088
+ "format": "f32-to-bf16",
1089
+ "nbytes": 33554432,
1090
+ "byteOffset": 0
1091
+ }
1092
+ ],
1093
+ "md5sum": "c32830c1ad07599365b35bdb97040713"
1094
+ },
1095
+ {
1096
+ "dataPath": "params_shard_27.bin",
1097
+ "format": "raw-shard",
1098
+ "nbytes": 25165824,
1099
+ "records": [
1100
+ {
1101
+ "name": "gpt_neox.layers.8.attention.query_key_value.weight",
1102
+ "shape": [
1103
+ 6144,
1104
+ 2048
1105
+ ],
1106
+ "dtype": "float16",
1107
+ "format": "f32-to-bf16",
1108
+ "nbytes": 25165824,
1109
+ "byteOffset": 0
1110
+ }
1111
+ ],
1112
+ "md5sum": "517ecd0e8410e75bd7b3be0f44d4f656"
1113
+ },
1114
+ {
1115
+ "dataPath": "params_shard_28.bin",
1116
+ "format": "raw-shard",
1117
+ "nbytes": 33554432,
1118
+ "records": [
1119
+ {
1120
+ "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.weight",
1121
+ "shape": [
1122
+ 8192,
1123
+ 2048
1124
+ ],
1125
+ "dtype": "float16",
1126
+ "format": "f32-to-bf16",
1127
+ "nbytes": 33554432,
1128
+ "byteOffset": 0
1129
+ }
1130
+ ],
1131
+ "md5sum": "4513e52799f3966b3090fe8f2c198570"
1132
+ },
1133
+ {
1134
+ "dataPath": "params_shard_29.bin",
1135
+ "format": "raw-shard",
1136
+ "nbytes": 33554432,
1137
+ "records": [
1138
+ {
1139
+ "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.weight",
1140
+ "shape": [
1141
+ 2048,
1142
+ 8192
1143
+ ],
1144
+ "dtype": "float16",
1145
+ "format": "f32-to-bf16",
1146
+ "nbytes": 33554432,
1147
+ "byteOffset": 0
1148
+ }
1149
+ ],
1150
+ "md5sum": "20f134a1a67f8da542a1594f8d145687"
1151
+ },
1152
+ {
1153
+ "dataPath": "params_shard_30.bin",
1154
+ "format": "raw-shard",
1155
+ "nbytes": 25165824,
1156
+ "records": [
1157
+ {
1158
+ "name": "gpt_neox.layers.9.attention.query_key_value.weight",
1159
+ "shape": [
1160
+ 6144,
1161
+ 2048
1162
+ ],
1163
+ "dtype": "float16",
1164
+ "format": "f32-to-bf16",
1165
+ "nbytes": 25165824,
1166
+ "byteOffset": 0
1167
+ }
1168
+ ],
1169
+ "md5sum": "7cda7ef64a1611eb36cff4f88fa70fd7"
1170
+ },
1171
+ {
1172
+ "dataPath": "params_shard_31.bin",
1173
+ "format": "raw-shard",
1174
+ "nbytes": 25325568,
1175
+ "records": [
1176
+ {
1177
+ "name": "gpt_neox.layers.6.attention.dense.weight",
1178
+ "shape": [
1179
+ 2048,
1180
+ 2048
1181
+ ],
1182
+ "dtype": "float16",
1183
+ "format": "f32-to-bf16",
1184
+ "nbytes": 8388608,
1185
+ "byteOffset": 0
1186
+ },
1187
+ {
1188
+ "name": "gpt_neox.layers.6.attention.dense.bias",
1189
+ "shape": [
1190
+ 2048
1191
+ ],
1192
+ "dtype": "float16",
1193
+ "format": "f32-to-bf16",
1194
+ "nbytes": 4096,
1195
+ "byteOffset": 8388608
1196
+ },
1197
+ {
1198
+ "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
1199
+ "shape": [
1200
+ 8192
1201
+ ],
1202
+ "dtype": "float32",
1203
+ "format": "f32-to-bf16",
1204
+ "nbytes": 16384,
1205
+ "byteOffset": 8392704
1206
+ },
1207
+ {
1208
+ "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
1209
+ "shape": [
1210
+ 2048
1211
+ ],
1212
+ "dtype": "float32",
1213
+ "format": "f32-to-bf16",
1214
+ "nbytes": 4096,
1215
+ "byteOffset": 8409088
1216
+ },
1217
+ {
1218
+ "name": "gpt_neox.layers.7.input_layernorm.weight",
1219
+ "shape": [
1220
+ 2048
1221
+ ],
1222
+ "dtype": "float16",
1223
+ "format": "f32-to-bf16",
1224
+ "nbytes": 4096,
1225
+ "byteOffset": 8413184
1226
+ },
1227
+ {
1228
+ "name": "gpt_neox.layers.7.input_layernorm.bias",
1229
+ "shape": [
1230
+ 2048
1231
+ ],
1232
+ "dtype": "float16",
1233
+ "format": "f32-to-bf16",
1234
+ "nbytes": 4096,
1235
+ "byteOffset": 8417280
1236
+ },
1237
+ {
1238
+ "name": "gpt_neox.layers.7.post_attention_layernorm.weight",
1239
+ "shape": [
1240
+ 2048
1241
+ ],
1242
+ "dtype": "float16",
1243
+ "format": "f32-to-bf16",
1244
+ "nbytes": 4096,
1245
+ "byteOffset": 8421376
1246
+ },
1247
+ {
1248
+ "name": "gpt_neox.layers.7.post_attention_layernorm.bias",
1249
+ "shape": [
1250
+ 2048
1251
+ ],
1252
+ "dtype": "float16",
1253
+ "format": "f32-to-bf16",
1254
+ "nbytes": 4096,
1255
+ "byteOffset": 8425472
1256
+ },
1257
+ {
1258
+ "name": "gpt_neox.layers.7.attention.query_key_value.bias",
1259
+ "shape": [
1260
+ 6144
1261
+ ],
1262
+ "dtype": "float16",
1263
+ "format": "f32-to-bf16",
1264
+ "nbytes": 12288,
1265
+ "byteOffset": 8429568
1266
+ },
1267
+ {
1268
+ "name": "gpt_neox.layers.7.attention.dense.weight",
1269
+ "shape": [
1270
+ 2048,
1271
+ 2048
1272
+ ],
1273
+ "dtype": "float16",
1274
+ "format": "f32-to-bf16",
1275
+ "nbytes": 8388608,
1276
+ "byteOffset": 8441856
1277
+ },
1278
+ {
1279
+ "name": "gpt_neox.layers.7.attention.dense.bias",
1280
+ "shape": [
1281
+ 2048
1282
+ ],
1283
+ "dtype": "float16",
1284
+ "format": "f32-to-bf16",
1285
+ "nbytes": 4096,
1286
+ "byteOffset": 16830464
1287
+ },
1288
+ {
1289
+ "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
1290
+ "shape": [
1291
+ 8192
1292
+ ],
1293
+ "dtype": "float32",
1294
+ "format": "f32-to-bf16",
1295
+ "nbytes": 16384,
1296
+ "byteOffset": 16834560
1297
+ },
1298
+ {
1299
+ "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
1300
+ "shape": [
1301
+ 2048
1302
+ ],
1303
+ "dtype": "float32",
1304
+ "format": "f32-to-bf16",
1305
+ "nbytes": 4096,
1306
+ "byteOffset": 16850944
1307
+ },
1308
+ {
1309
+ "name": "gpt_neox.layers.8.input_layernorm.weight",
1310
+ "shape": [
1311
+ 2048
1312
+ ],
1313
+ "dtype": "float16",
1314
+ "format": "f32-to-bf16",
1315
+ "nbytes": 4096,
1316
+ "byteOffset": 16855040
1317
+ },
1318
+ {
1319
+ "name": "gpt_neox.layers.8.input_layernorm.bias",
1320
+ "shape": [
1321
+ 2048
1322
+ ],
1323
+ "dtype": "float16",
1324
+ "format": "f32-to-bf16",
1325
+ "nbytes": 4096,
1326
+ "byteOffset": 16859136
1327
+ },
1328
+ {
1329
+ "name": "gpt_neox.layers.8.post_attention_layernorm.weight",
1330
+ "shape": [
1331
+ 2048
1332
+ ],
1333
+ "dtype": "float16",
1334
+ "format": "f32-to-bf16",
1335
+ "nbytes": 4096,
1336
+ "byteOffset": 16863232
1337
+ },
1338
+ {
1339
+ "name": "gpt_neox.layers.8.post_attention_layernorm.bias",
1340
+ "shape": [
1341
+ 2048
1342
+ ],
1343
+ "dtype": "float16",
1344
+ "format": "f32-to-bf16",
1345
+ "nbytes": 4096,
1346
+ "byteOffset": 16867328
1347
+ },
1348
+ {
1349
+ "name": "gpt_neox.layers.8.attention.query_key_value.bias",
1350
+ "shape": [
1351
+ 6144
1352
+ ],
1353
+ "dtype": "float16",
1354
+ "format": "f32-to-bf16",
1355
+ "nbytes": 12288,
1356
+ "byteOffset": 16871424
1357
+ },
1358
+ {
1359
+ "name": "gpt_neox.layers.8.attention.dense.weight",
1360
+ "shape": [
1361
+ 2048,
1362
+ 2048
1363
+ ],
1364
+ "dtype": "float16",
1365
+ "format": "f32-to-bf16",
1366
+ "nbytes": 8388608,
1367
+ "byteOffset": 16883712
1368
+ },
1369
+ {
1370
+ "name": "gpt_neox.layers.8.attention.dense.bias",
1371
+ "shape": [
1372
+ 2048
1373
+ ],
1374
+ "dtype": "float16",
1375
+ "format": "f32-to-bf16",
1376
+ "nbytes": 4096,
1377
+ "byteOffset": 25272320
1378
+ },
1379
+ {
1380
+ "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
1381
+ "shape": [
1382
+ 8192
1383
+ ],
1384
+ "dtype": "float32",
1385
+ "format": "f32-to-bf16",
1386
+ "nbytes": 16384,
1387
+ "byteOffset": 25276416
1388
+ },
1389
+ {
1390
+ "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
1391
+ "shape": [
1392
+ 2048
1393
+ ],
1394
+ "dtype": "float32",
1395
+ "format": "f32-to-bf16",
1396
+ "nbytes": 4096,
1397
+ "byteOffset": 25292800
1398
+ },
1399
+ {
1400
+ "name": "gpt_neox.layers.9.input_layernorm.weight",
1401
+ "shape": [
1402
+ 2048
1403
+ ],
1404
+ "dtype": "float16",
1405
+ "format": "f32-to-bf16",
1406
+ "nbytes": 4096,
1407
+ "byteOffset": 25296896
1408
+ },
1409
+ {
1410
+ "name": "gpt_neox.layers.9.input_layernorm.bias",
1411
+ "shape": [
1412
+ 2048
1413
+ ],
1414
+ "dtype": "float16",
1415
+ "format": "f32-to-bf16",
1416
+ "nbytes": 4096,
1417
+ "byteOffset": 25300992
1418
+ },
1419
+ {
1420
+ "name": "gpt_neox.layers.9.post_attention_layernorm.weight",
1421
+ "shape": [
1422
+ 2048
1423
+ ],
1424
+ "dtype": "float16",
1425
+ "format": "f32-to-bf16",
1426
+ "nbytes": 4096,
1427
+ "byteOffset": 25305088
1428
+ },
1429
+ {
1430
+ "name": "gpt_neox.layers.9.post_attention_layernorm.bias",
1431
+ "shape": [
1432
+ 2048
1433
+ ],
1434
+ "dtype": "float16",
1435
+ "format": "f32-to-bf16",
1436
+ "nbytes": 4096,
1437
+ "byteOffset": 25309184
1438
+ },
1439
+ {
1440
+ "name": "gpt_neox.layers.9.attention.query_key_value.bias",
1441
+ "shape": [
1442
+ 6144
1443
+ ],
1444
+ "dtype": "float16",
1445
+ "format": "f32-to-bf16",
1446
+ "nbytes": 12288,
1447
+ "byteOffset": 25313280
1448
+ }
1449
+ ],
1450
+ "md5sum": "e259ef76f92def9684153bc75388edae"
1451
+ },
1452
+ {
1453
+ "dataPath": "params_shard_32.bin",
1454
+ "format": "raw-shard",
1455
+ "nbytes": 33554432,
1456
+ "records": [
1457
+ {
1458
+ "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.weight",
1459
+ "shape": [
1460
+ 8192,
1461
+ 2048
1462
+ ],
1463
+ "dtype": "float16",
1464
+ "format": "f32-to-bf16",
1465
+ "nbytes": 33554432,
1466
+ "byteOffset": 0
1467
+ }
1468
+ ],
1469
+ "md5sum": "4e2fa9d7b430b8093610199bacc9e7a8"
1470
+ },
1471
+ {
1472
+ "dataPath": "params_shard_33.bin",
1473
+ "format": "raw-shard",
1474
+ "nbytes": 33554432,
1475
+ "records": [
1476
+ {
1477
+ "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.weight",
1478
+ "shape": [
1479
+ 2048,
1480
+ 8192
1481
+ ],
1482
+ "dtype": "float16",
1483
+ "format": "f32-to-bf16",
1484
+ "nbytes": 33554432,
1485
+ "byteOffset": 0
1486
+ }
1487
+ ],
1488
+ "md5sum": "833726affdd62d945fea452daa4be81e"
1489
+ },
1490
+ {
1491
+ "dataPath": "params_shard_34.bin",
1492
+ "format": "raw-shard",
1493
+ "nbytes": 25165824,
1494
+ "records": [
1495
+ {
1496
+ "name": "gpt_neox.layers.10.attention.query_key_value.weight",
1497
+ "shape": [
1498
+ 6144,
1499
+ 2048
1500
+ ],
1501
+ "dtype": "float16",
1502
+ "format": "f32-to-bf16",
1503
+ "nbytes": 25165824,
1504
+ "byteOffset": 0
1505
+ }
1506
+ ],
1507
+ "md5sum": "a90521567e0f31a50ab8ee830a33ec11"
1508
+ },
1509
+ {
1510
+ "dataPath": "params_shard_35.bin",
1511
+ "format": "raw-shard",
1512
+ "nbytes": 33554432,
1513
+ "records": [
1514
+ {
1515
+ "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.weight",
1516
+ "shape": [
1517
+ 8192,
1518
+ 2048
1519
+ ],
1520
+ "dtype": "float16",
1521
+ "format": "f32-to-bf16",
1522
+ "nbytes": 33554432,
1523
+ "byteOffset": 0
1524
+ }
1525
+ ],
1526
+ "md5sum": "2017f8b0fcce69d68a36eddf4f065fb6"
1527
+ },
1528
+ {
1529
+ "dataPath": "params_shard_36.bin",
1530
+ "format": "raw-shard",
1531
+ "nbytes": 33554432,
1532
+ "records": [
1533
+ {
1534
+ "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.weight",
1535
+ "shape": [
1536
+ 2048,
1537
+ 8192
1538
+ ],
1539
+ "dtype": "float16",
1540
+ "format": "f32-to-bf16",
1541
+ "nbytes": 33554432,
1542
+ "byteOffset": 0
1543
+ }
1544
+ ],
1545
+ "md5sum": "1bb0df0f78e29b562fb80e9416f2e6d0"
1546
+ },
1547
+ {
1548
+ "dataPath": "params_shard_37.bin",
1549
+ "format": "raw-shard",
1550
+ "nbytes": 25165824,
1551
+ "records": [
1552
+ {
1553
+ "name": "gpt_neox.layers.11.attention.query_key_value.weight",
1554
+ "shape": [
1555
+ 6144,
1556
+ 2048
1557
+ ],
1558
+ "dtype": "float16",
1559
+ "format": "f32-to-bf16",
1560
+ "nbytes": 25165824,
1561
+ "byteOffset": 0
1562
+ }
1563
+ ],
1564
+ "md5sum": "c5bcae6dd378facfbb316629a353996e"
1565
+ },
1566
+ {
1567
+ "dataPath": "params_shard_38.bin",
1568
+ "format": "raw-shard",
1569
+ "nbytes": 33554432,
1570
+ "records": [
1571
+ {
1572
+ "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.weight",
1573
+ "shape": [
1574
+ 8192,
1575
+ 2048
1576
+ ],
1577
+ "dtype": "float16",
1578
+ "format": "f32-to-bf16",
1579
+ "nbytes": 33554432,
1580
+ "byteOffset": 0
1581
+ }
1582
+ ],
1583
+ "md5sum": "77116769d32f06f166da36ca2a268c64"
1584
+ },
1585
+ {
1586
+ "dataPath": "params_shard_39.bin",
1587
+ "format": "raw-shard",
1588
+ "nbytes": 33554432,
1589
+ "records": [
1590
+ {
1591
+ "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.weight",
1592
+ "shape": [
1593
+ 2048,
1594
+ 8192
1595
+ ],
1596
+ "dtype": "float16",
1597
+ "format": "f32-to-bf16",
1598
+ "nbytes": 33554432,
1599
+ "byteOffset": 0
1600
+ }
1601
+ ],
1602
+ "md5sum": "b9e187b32f191e5cd83521eb59d918fd"
1603
+ },
1604
+ {
1605
+ "dataPath": "params_shard_40.bin",
1606
+ "format": "raw-shard",
1607
+ "nbytes": 25165824,
1608
+ "records": [
1609
+ {
1610
+ "name": "gpt_neox.layers.12.attention.query_key_value.weight",
1611
+ "shape": [
1612
+ 6144,
1613
+ 2048
1614
+ ],
1615
+ "dtype": "float16",
1616
+ "format": "f32-to-bf16",
1617
+ "nbytes": 25165824,
1618
+ "byteOffset": 0
1619
+ }
1620
+ ],
1621
+ "md5sum": "010608c5a52ef822e207474b2f209681"
1622
+ },
1623
+ {
1624
+ "dataPath": "params_shard_41.bin",
1625
+ "format": "raw-shard",
1626
+ "nbytes": 25325568,
1627
+ "records": [
1628
+ {
1629
+ "name": "gpt_neox.layers.9.attention.dense.weight",
1630
+ "shape": [
1631
+ 2048,
1632
+ 2048
1633
+ ],
1634
+ "dtype": "float16",
1635
+ "format": "f32-to-bf16",
1636
+ "nbytes": 8388608,
1637
+ "byteOffset": 0
1638
+ },
1639
+ {
1640
+ "name": "gpt_neox.layers.9.attention.dense.bias",
1641
+ "shape": [
1642
+ 2048
1643
+ ],
1644
+ "dtype": "float16",
1645
+ "format": "f32-to-bf16",
1646
+ "nbytes": 4096,
1647
+ "byteOffset": 8388608
1648
+ },
1649
+ {
1650
+ "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
1651
+ "shape": [
1652
+ 8192
1653
+ ],
1654
+ "dtype": "float32",
1655
+ "format": "f32-to-bf16",
1656
+ "nbytes": 16384,
1657
+ "byteOffset": 8392704
1658
+ },
1659
+ {
1660
+ "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
1661
+ "shape": [
1662
+ 2048
1663
+ ],
1664
+ "dtype": "float32",
1665
+ "format": "f32-to-bf16",
1666
+ "nbytes": 4096,
1667
+ "byteOffset": 8409088
1668
+ },
1669
+ {
1670
+ "name": "gpt_neox.layers.10.input_layernorm.weight",
1671
+ "shape": [
1672
+ 2048
1673
+ ],
1674
+ "dtype": "float16",
1675
+ "format": "f32-to-bf16",
1676
+ "nbytes": 4096,
1677
+ "byteOffset": 8413184
1678
+ },
1679
+ {
1680
+ "name": "gpt_neox.layers.10.input_layernorm.bias",
1681
+ "shape": [
1682
+ 2048
1683
+ ],
1684
+ "dtype": "float16",
1685
+ "format": "f32-to-bf16",
1686
+ "nbytes": 4096,
1687
+ "byteOffset": 8417280
1688
+ },
1689
+ {
1690
+ "name": "gpt_neox.layers.10.post_attention_layernorm.weight",
1691
+ "shape": [
1692
+ 2048
1693
+ ],
1694
+ "dtype": "float16",
1695
+ "format": "f32-to-bf16",
1696
+ "nbytes": 4096,
1697
+ "byteOffset": 8421376
1698
+ },
1699
+ {
1700
+ "name": "gpt_neox.layers.10.post_attention_layernorm.bias",
1701
+ "shape": [
1702
+ 2048
1703
+ ],
1704
+ "dtype": "float16",
1705
+ "format": "f32-to-bf16",
1706
+ "nbytes": 4096,
1707
+ "byteOffset": 8425472
1708
+ },
1709
+ {
1710
+ "name": "gpt_neox.layers.10.attention.query_key_value.bias",
1711
+ "shape": [
1712
+ 6144
1713
+ ],
1714
+ "dtype": "float16",
1715
+ "format": "f32-to-bf16",
1716
+ "nbytes": 12288,
1717
+ "byteOffset": 8429568
1718
+ },
1719
+ {
1720
+ "name": "gpt_neox.layers.10.attention.dense.weight",
1721
+ "shape": [
1722
+ 2048,
1723
+ 2048
1724
+ ],
1725
+ "dtype": "float16",
1726
+ "format": "f32-to-bf16",
1727
+ "nbytes": 8388608,
1728
+ "byteOffset": 8441856
1729
+ },
1730
+ {
1731
+ "name": "gpt_neox.layers.10.attention.dense.bias",
1732
+ "shape": [
1733
+ 2048
1734
+ ],
1735
+ "dtype": "float16",
1736
+ "format": "f32-to-bf16",
1737
+ "nbytes": 4096,
1738
+ "byteOffset": 16830464
1739
+ },
1740
+ {
1741
+ "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
1742
+ "shape": [
1743
+ 8192
1744
+ ],
1745
+ "dtype": "float32",
1746
+ "format": "f32-to-bf16",
1747
+ "nbytes": 16384,
1748
+ "byteOffset": 16834560
1749
+ },
1750
+ {
1751
+ "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
1752
+ "shape": [
1753
+ 2048
1754
+ ],
1755
+ "dtype": "float32",
1756
+ "format": "f32-to-bf16",
1757
+ "nbytes": 4096,
1758
+ "byteOffset": 16850944
1759
+ },
1760
+ {
1761
+ "name": "gpt_neox.layers.11.input_layernorm.weight",
1762
+ "shape": [
1763
+ 2048
1764
+ ],
1765
+ "dtype": "float16",
1766
+ "format": "f32-to-bf16",
1767
+ "nbytes": 4096,
1768
+ "byteOffset": 16855040
1769
+ },
1770
+ {
1771
+ "name": "gpt_neox.layers.11.input_layernorm.bias",
1772
+ "shape": [
1773
+ 2048
1774
+ ],
1775
+ "dtype": "float16",
1776
+ "format": "f32-to-bf16",
1777
+ "nbytes": 4096,
1778
+ "byteOffset": 16859136
1779
+ },
1780
+ {
1781
+ "name": "gpt_neox.layers.11.post_attention_layernorm.weight",
1782
+ "shape": [
1783
+ 2048
1784
+ ],
1785
+ "dtype": "float16",
1786
+ "format": "f32-to-bf16",
1787
+ "nbytes": 4096,
1788
+ "byteOffset": 16863232
1789
+ },
1790
+ {
1791
+ "name": "gpt_neox.layers.11.post_attention_layernorm.bias",
1792
+ "shape": [
1793
+ 2048
1794
+ ],
1795
+ "dtype": "float16",
1796
+ "format": "f32-to-bf16",
1797
+ "nbytes": 4096,
1798
+ "byteOffset": 16867328
1799
+ },
1800
+ {
1801
+ "name": "gpt_neox.layers.11.attention.query_key_value.bias",
1802
+ "shape": [
1803
+ 6144
1804
+ ],
1805
+ "dtype": "float16",
1806
+ "format": "f32-to-bf16",
1807
+ "nbytes": 12288,
1808
+ "byteOffset": 16871424
1809
+ },
1810
+ {
1811
+ "name": "gpt_neox.layers.11.attention.dense.weight",
1812
+ "shape": [
1813
+ 2048,
1814
+ 2048
1815
+ ],
1816
+ "dtype": "float16",
1817
+ "format": "f32-to-bf16",
1818
+ "nbytes": 8388608,
1819
+ "byteOffset": 16883712
1820
+ },
1821
+ {
1822
+ "name": "gpt_neox.layers.11.attention.dense.bias",
1823
+ "shape": [
1824
+ 2048
1825
+ ],
1826
+ "dtype": "float16",
1827
+ "format": "f32-to-bf16",
1828
+ "nbytes": 4096,
1829
+ "byteOffset": 25272320
1830
+ },
1831
+ {
1832
+ "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
1833
+ "shape": [
1834
+ 8192
1835
+ ],
1836
+ "dtype": "float32",
1837
+ "format": "f32-to-bf16",
1838
+ "nbytes": 16384,
1839
+ "byteOffset": 25276416
1840
+ },
1841
+ {
1842
+ "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
1843
+ "shape": [
1844
+ 2048
1845
+ ],
1846
+ "dtype": "float32",
1847
+ "format": "f32-to-bf16",
1848
+ "nbytes": 4096,
1849
+ "byteOffset": 25292800
1850
+ },
1851
+ {
1852
+ "name": "gpt_neox.layers.12.input_layernorm.weight",
1853
+ "shape": [
1854
+ 2048
1855
+ ],
1856
+ "dtype": "float16",
1857
+ "format": "f32-to-bf16",
1858
+ "nbytes": 4096,
1859
+ "byteOffset": 25296896
1860
+ },
1861
+ {
1862
+ "name": "gpt_neox.layers.12.input_layernorm.bias",
1863
+ "shape": [
1864
+ 2048
1865
+ ],
1866
+ "dtype": "float16",
1867
+ "format": "f32-to-bf16",
1868
+ "nbytes": 4096,
1869
+ "byteOffset": 25300992
1870
+ },
1871
+ {
1872
+ "name": "gpt_neox.layers.12.post_attention_layernorm.weight",
1873
+ "shape": [
1874
+ 2048
1875
+ ],
1876
+ "dtype": "float16",
1877
+ "format": "f32-to-bf16",
1878
+ "nbytes": 4096,
1879
+ "byteOffset": 25305088
1880
+ },
1881
+ {
1882
+ "name": "gpt_neox.layers.12.post_attention_layernorm.bias",
1883
+ "shape": [
1884
+ 2048
1885
+ ],
1886
+ "dtype": "float16",
1887
+ "format": "f32-to-bf16",
1888
+ "nbytes": 4096,
1889
+ "byteOffset": 25309184
1890
+ },
1891
+ {
1892
+ "name": "gpt_neox.layers.12.attention.query_key_value.bias",
1893
+ "shape": [
1894
+ 6144
1895
+ ],
1896
+ "dtype": "float16",
1897
+ "format": "f32-to-bf16",
1898
+ "nbytes": 12288,
1899
+ "byteOffset": 25313280
1900
+ }
1901
+ ],
1902
+ "md5sum": "9d0c8914ae6f3f6fcfb66ac13eb4d0a3"
1903
+ },
1904
+ {
1905
+ "dataPath": "params_shard_42.bin",
1906
+ "format": "raw-shard",
1907
+ "nbytes": 33554432,
1908
+ "records": [
1909
+ {
1910
+ "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.weight",
1911
+ "shape": [
1912
+ 8192,
1913
+ 2048
1914
+ ],
1915
+ "dtype": "float16",
1916
+ "format": "f32-to-bf16",
1917
+ "nbytes": 33554432,
1918
+ "byteOffset": 0
1919
+ }
1920
+ ],
1921
+ "md5sum": "ab87a989c27431535f15efa4e031aad7"
1922
+ },
1923
+ {
1924
+ "dataPath": "params_shard_43.bin",
1925
+ "format": "raw-shard",
1926
+ "nbytes": 33554432,
1927
+ "records": [
1928
+ {
1929
+ "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.weight",
1930
+ "shape": [
1931
+ 2048,
1932
+ 8192
1933
+ ],
1934
+ "dtype": "float16",
1935
+ "format": "f32-to-bf16",
1936
+ "nbytes": 33554432,
1937
+ "byteOffset": 0
1938
+ }
1939
+ ],
1940
+ "md5sum": "0dfda91cfe8ba98fff5a402cfe1dc7df"
1941
+ },
1942
+ {
1943
+ "dataPath": "params_shard_44.bin",
1944
+ "format": "raw-shard",
1945
+ "nbytes": 25165824,
1946
+ "records": [
1947
+ {
1948
+ "name": "gpt_neox.layers.13.attention.query_key_value.weight",
1949
+ "shape": [
1950
+ 6144,
1951
+ 2048
1952
+ ],
1953
+ "dtype": "float16",
1954
+ "format": "f32-to-bf16",
1955
+ "nbytes": 25165824,
1956
+ "byteOffset": 0
1957
+ }
1958
+ ],
1959
+ "md5sum": "287808f7646050482dd9d4f73f3d71e1"
1960
+ },
1961
+ {
1962
+ "dataPath": "params_shard_45.bin",
1963
+ "format": "raw-shard",
1964
+ "nbytes": 33554432,
1965
+ "records": [
1966
+ {
1967
+ "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.weight",
1968
+ "shape": [
1969
+ 8192,
1970
+ 2048
1971
+ ],
1972
+ "dtype": "float16",
1973
+ "format": "f32-to-bf16",
1974
+ "nbytes": 33554432,
1975
+ "byteOffset": 0
1976
+ }
1977
+ ],
1978
+ "md5sum": "d556cb9c06f1ffaf060b18a650d7eec2"
1979
+ },
1980
+ {
1981
+ "dataPath": "params_shard_46.bin",
1982
+ "format": "raw-shard",
1983
+ "nbytes": 33554432,
1984
+ "records": [
1985
+ {
1986
+ "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.weight",
1987
+ "shape": [
1988
+ 2048,
1989
+ 8192
1990
+ ],
1991
+ "dtype": "float16",
1992
+ "format": "f32-to-bf16",
1993
+ "nbytes": 33554432,
1994
+ "byteOffset": 0
1995
+ }
1996
+ ],
1997
+ "md5sum": "c2e914507fcbf57fa0f6e2c0d0803550"
1998
+ },
1999
+ {
2000
+ "dataPath": "params_shard_47.bin",
2001
+ "format": "raw-shard",
2002
+ "nbytes": 25165824,
2003
+ "records": [
2004
+ {
2005
+ "name": "gpt_neox.layers.14.attention.query_key_value.weight",
2006
+ "shape": [
2007
+ 6144,
2008
+ 2048
2009
+ ],
2010
+ "dtype": "float16",
2011
+ "format": "f32-to-bf16",
2012
+ "nbytes": 25165824,
2013
+ "byteOffset": 0
2014
+ }
2015
+ ],
2016
+ "md5sum": "6dc1cb91a703d693796b3135b7a95ff6"
2017
+ },
2018
+ {
2019
+ "dataPath": "params_shard_48.bin",
2020
+ "format": "raw-shard",
2021
+ "nbytes": 33554432,
2022
+ "records": [
2023
+ {
2024
+ "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.weight",
2025
+ "shape": [
2026
+ 8192,
2027
+ 2048
2028
+ ],
2029
+ "dtype": "float16",
2030
+ "format": "f32-to-bf16",
2031
+ "nbytes": 33554432,
2032
+ "byteOffset": 0
2033
+ }
2034
+ ],
2035
+ "md5sum": "9905cbb2b08b5ffbc8b154005dd5ed0c"
2036
+ },
2037
+ {
2038
+ "dataPath": "params_shard_49.bin",
2039
+ "format": "raw-shard",
2040
+ "nbytes": 33554432,
2041
+ "records": [
2042
+ {
2043
+ "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.weight",
2044
+ "shape": [
2045
+ 2048,
2046
+ 8192
2047
+ ],
2048
+ "dtype": "float16",
2049
+ "format": "f32-to-bf16",
2050
+ "nbytes": 33554432,
2051
+ "byteOffset": 0
2052
+ }
2053
+ ],
2054
+ "md5sum": "62120c836ef22bd90e4391cd148ccd2f"
2055
+ },
2056
+ {
2057
+ "dataPath": "params_shard_50.bin",
2058
+ "format": "raw-shard",
2059
+ "nbytes": 25165824,
2060
+ "records": [
2061
+ {
2062
+ "name": "gpt_neox.layers.15.attention.query_key_value.weight",
2063
+ "shape": [
2064
+ 6144,
2065
+ 2048
2066
+ ],
2067
+ "dtype": "float16",
2068
+ "format": "f32-to-bf16",
2069
+ "nbytes": 25165824,
2070
+ "byteOffset": 0
2071
+ }
2072
+ ],
2073
+ "md5sum": "f3756f5a5824708fd51f3de6e54449e5"
2074
+ },
2075
+ {
2076
+ "dataPath": "params_shard_51.bin",
2077
+ "format": "raw-shard",
2078
+ "nbytes": 25325568,
2079
+ "records": [
2080
+ {
2081
+ "name": "gpt_neox.layers.12.attention.dense.weight",
2082
+ "shape": [
2083
+ 2048,
2084
+ 2048
2085
+ ],
2086
+ "dtype": "float16",
2087
+ "format": "f32-to-bf16",
2088
+ "nbytes": 8388608,
2089
+ "byteOffset": 0
2090
+ },
2091
+ {
2092
+ "name": "gpt_neox.layers.12.attention.dense.bias",
2093
+ "shape": [
2094
+ 2048
2095
+ ],
2096
+ "dtype": "float16",
2097
+ "format": "f32-to-bf16",
2098
+ "nbytes": 4096,
2099
+ "byteOffset": 8388608
2100
+ },
2101
+ {
2102
+ "name": "gpt_neox.layers.12.mlp.dense_h_to_4h.bias",
2103
+ "shape": [
2104
+ 8192
2105
+ ],
2106
+ "dtype": "float32",
2107
+ "format": "f32-to-bf16",
2108
+ "nbytes": 16384,
2109
+ "byteOffset": 8392704
2110
+ },
2111
+ {
2112
+ "name": "gpt_neox.layers.12.mlp.dense_4h_to_h.bias",
2113
+ "shape": [
2114
+ 2048
2115
+ ],
2116
+ "dtype": "float32",
2117
+ "format": "f32-to-bf16",
2118
+ "nbytes": 4096,
2119
+ "byteOffset": 8409088
2120
+ },
2121
+ {
2122
+ "name": "gpt_neox.layers.13.input_layernorm.weight",
2123
+ "shape": [
2124
+ 2048
2125
+ ],
2126
+ "dtype": "float16",
2127
+ "format": "f32-to-bf16",
2128
+ "nbytes": 4096,
2129
+ "byteOffset": 8413184
2130
+ },
2131
+ {
2132
+ "name": "gpt_neox.layers.13.input_layernorm.bias",
2133
+ "shape": [
2134
+ 2048
2135
+ ],
2136
+ "dtype": "float16",
2137
+ "format": "f32-to-bf16",
2138
+ "nbytes": 4096,
2139
+ "byteOffset": 8417280
2140
+ },
2141
+ {
2142
+ "name": "gpt_neox.layers.13.post_attention_layernorm.weight",
2143
+ "shape": [
2144
+ 2048
2145
+ ],
2146
+ "dtype": "float16",
2147
+ "format": "f32-to-bf16",
2148
+ "nbytes": 4096,
2149
+ "byteOffset": 8421376
2150
+ },
2151
+ {
2152
+ "name": "gpt_neox.layers.13.post_attention_layernorm.bias",
2153
+ "shape": [
2154
+ 2048
2155
+ ],
2156
+ "dtype": "float16",
2157
+ "format": "f32-to-bf16",
2158
+ "nbytes": 4096,
2159
+ "byteOffset": 8425472
2160
+ },
2161
+ {
2162
+ "name": "gpt_neox.layers.13.attention.query_key_value.bias",
2163
+ "shape": [
2164
+ 6144
2165
+ ],
2166
+ "dtype": "float16",
2167
+ "format": "f32-to-bf16",
2168
+ "nbytes": 12288,
2169
+ "byteOffset": 8429568
2170
+ },
2171
+ {
2172
+ "name": "gpt_neox.layers.13.attention.dense.weight",
2173
+ "shape": [
2174
+ 2048,
2175
+ 2048
2176
+ ],
2177
+ "dtype": "float16",
2178
+ "format": "f32-to-bf16",
2179
+ "nbytes": 8388608,
2180
+ "byteOffset": 8441856
2181
+ },
2182
+ {
2183
+ "name": "gpt_neox.layers.13.attention.dense.bias",
2184
+ "shape": [
2185
+ 2048
2186
+ ],
2187
+ "dtype": "float16",
2188
+ "format": "f32-to-bf16",
2189
+ "nbytes": 4096,
2190
+ "byteOffset": 16830464
2191
+ },
2192
+ {
2193
+ "name": "gpt_neox.layers.13.mlp.dense_h_to_4h.bias",
2194
+ "shape": [
2195
+ 8192
2196
+ ],
2197
+ "dtype": "float32",
2198
+ "format": "f32-to-bf16",
2199
+ "nbytes": 16384,
2200
+ "byteOffset": 16834560
2201
+ },
2202
+ {
2203
+ "name": "gpt_neox.layers.13.mlp.dense_4h_to_h.bias",
2204
+ "shape": [
2205
+ 2048
2206
+ ],
2207
+ "dtype": "float32",
2208
+ "format": "f32-to-bf16",
2209
+ "nbytes": 4096,
2210
+ "byteOffset": 16850944
2211
+ },
2212
+ {
2213
+ "name": "gpt_neox.layers.14.input_layernorm.weight",
2214
+ "shape": [
2215
+ 2048
2216
+ ],
2217
+ "dtype": "float16",
2218
+ "format": "f32-to-bf16",
2219
+ "nbytes": 4096,
2220
+ "byteOffset": 16855040
2221
+ },
2222
+ {
2223
+ "name": "gpt_neox.layers.14.input_layernorm.bias",
2224
+ "shape": [
2225
+ 2048
2226
+ ],
2227
+ "dtype": "float16",
2228
+ "format": "f32-to-bf16",
2229
+ "nbytes": 4096,
2230
+ "byteOffset": 16859136
2231
+ },
2232
+ {
2233
+ "name": "gpt_neox.layers.14.post_attention_layernorm.weight",
2234
+ "shape": [
2235
+ 2048
2236
+ ],
2237
+ "dtype": "float16",
2238
+ "format": "f32-to-bf16",
2239
+ "nbytes": 4096,
2240
+ "byteOffset": 16863232
2241
+ },
2242
+ {
2243
+ "name": "gpt_neox.layers.14.post_attention_layernorm.bias",
2244
+ "shape": [
2245
+ 2048
2246
+ ],
2247
+ "dtype": "float16",
2248
+ "format": "f32-to-bf16",
2249
+ "nbytes": 4096,
2250
+ "byteOffset": 16867328
2251
+ },
2252
+ {
2253
+ "name": "gpt_neox.layers.14.attention.query_key_value.bias",
2254
+ "shape": [
2255
+ 6144
2256
+ ],
2257
+ "dtype": "float16",
2258
+ "format": "f32-to-bf16",
2259
+ "nbytes": 12288,
2260
+ "byteOffset": 16871424
2261
+ },
2262
+ {
2263
+ "name": "gpt_neox.layers.14.attention.dense.weight",
2264
+ "shape": [
2265
+ 2048,
2266
+ 2048
2267
+ ],
2268
+ "dtype": "float16",
2269
+ "format": "f32-to-bf16",
2270
+ "nbytes": 8388608,
2271
+ "byteOffset": 16883712
2272
+ },
2273
+ {
2274
+ "name": "gpt_neox.layers.14.attention.dense.bias",
2275
+ "shape": [
2276
+ 2048
2277
+ ],
2278
+ "dtype": "float16",
2279
+ "format": "f32-to-bf16",
2280
+ "nbytes": 4096,
2281
+ "byteOffset": 25272320
2282
+ },
2283
+ {
2284
+ "name": "gpt_neox.layers.14.mlp.dense_h_to_4h.bias",
2285
+ "shape": [
2286
+ 8192
2287
+ ],
2288
+ "dtype": "float32",
2289
+ "format": "f32-to-bf16",
2290
+ "nbytes": 16384,
2291
+ "byteOffset": 25276416
2292
+ },
2293
+ {
2294
+ "name": "gpt_neox.layers.14.mlp.dense_4h_to_h.bias",
2295
+ "shape": [
2296
+ 2048
2297
+ ],
2298
+ "dtype": "float32",
2299
+ "format": "f32-to-bf16",
2300
+ "nbytes": 4096,
2301
+ "byteOffset": 25292800
2302
+ },
2303
+ {
2304
+ "name": "gpt_neox.layers.15.input_layernorm.weight",
2305
+ "shape": [
2306
+ 2048
2307
+ ],
2308
+ "dtype": "float16",
2309
+ "format": "f32-to-bf16",
2310
+ "nbytes": 4096,
2311
+ "byteOffset": 25296896
2312
+ },
2313
+ {
2314
+ "name": "gpt_neox.layers.15.input_layernorm.bias",
2315
+ "shape": [
2316
+ 2048
2317
+ ],
2318
+ "dtype": "float16",
2319
+ "format": "f32-to-bf16",
2320
+ "nbytes": 4096,
2321
+ "byteOffset": 25300992
2322
+ },
2323
+ {
2324
+ "name": "gpt_neox.layers.15.post_attention_layernorm.weight",
2325
+ "shape": [
2326
+ 2048
2327
+ ],
2328
+ "dtype": "float16",
2329
+ "format": "f32-to-bf16",
2330
+ "nbytes": 4096,
2331
+ "byteOffset": 25305088
2332
+ },
2333
+ {
2334
+ "name": "gpt_neox.layers.15.post_attention_layernorm.bias",
2335
+ "shape": [
2336
+ 2048
2337
+ ],
2338
+ "dtype": "float16",
2339
+ "format": "f32-to-bf16",
2340
+ "nbytes": 4096,
2341
+ "byteOffset": 25309184
2342
+ },
2343
+ {
2344
+ "name": "gpt_neox.layers.15.attention.query_key_value.bias",
2345
+ "shape": [
2346
+ 6144
2347
+ ],
2348
+ "dtype": "float16",
2349
+ "format": "f32-to-bf16",
2350
+ "nbytes": 12288,
2351
+ "byteOffset": 25313280
2352
+ }
2353
+ ],
2354
+ "md5sum": "b7ae6066328d6d58cdba7752718dea43"
2355
+ },
2356
+ {
2357
+ "dataPath": "params_shard_52.bin",
2358
+ "format": "raw-shard",
2359
+ "nbytes": 33554432,
2360
+ "records": [
2361
+ {
2362
+ "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.weight",
2363
+ "shape": [
2364
+ 8192,
2365
+ 2048
2366
+ ],
2367
+ "dtype": "float16",
2368
+ "format": "f32-to-bf16",
2369
+ "nbytes": 33554432,
2370
+ "byteOffset": 0
2371
+ }
2372
+ ],
2373
+ "md5sum": "362174f589aa7225e0ed1396c68b612b"
2374
+ },
2375
+ {
2376
+ "dataPath": "params_shard_53.bin",
2377
+ "format": "raw-shard",
2378
+ "nbytes": 33554432,
2379
+ "records": [
2380
+ {
2381
+ "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.weight",
2382
+ "shape": [
2383
+ 2048,
2384
+ 8192
2385
+ ],
2386
+ "dtype": "float16",
2387
+ "format": "f32-to-bf16",
2388
+ "nbytes": 33554432,
2389
+ "byteOffset": 0
2390
+ }
2391
+ ],
2392
+ "md5sum": "abb280cf7c20e138d18a8f9cf85322c8"
2393
+ },
2394
+ {
2395
+ "dataPath": "params_shard_54.bin",
2396
+ "format": "raw-shard",
2397
+ "nbytes": 206045184,
2398
+ "records": [
2399
+ {
2400
+ "name": "embed_out.weight",
2401
+ "shape": [
2402
+ 50304,
2403
+ 2048
2404
+ ],
2405
+ "dtype": "float16",
2406
+ "format": "f32-to-bf16",
2407
+ "nbytes": 206045184,
2408
+ "byteOffset": 0
2409
+ }
2410
+ ],
2411
+ "md5sum": "3939e915eb3bd60919c8b6d517ea99f6"
2412
+ },
2413
+ {
2414
+ "dataPath": "params_shard_55.bin",
2415
+ "format": "raw-shard",
2416
+ "nbytes": 8421376,
2417
+ "records": [
2418
+ {
2419
+ "name": "gpt_neox.layers.15.attention.dense.weight",
2420
+ "shape": [
2421
+ 2048,
2422
+ 2048
2423
+ ],
2424
+ "dtype": "float16",
2425
+ "format": "f32-to-bf16",
2426
+ "nbytes": 8388608,
2427
+ "byteOffset": 0
2428
+ },
2429
+ {
2430
+ "name": "gpt_neox.layers.15.attention.dense.bias",
2431
+ "shape": [
2432
+ 2048
2433
+ ],
2434
+ "dtype": "float16",
2435
+ "format": "f32-to-bf16",
2436
+ "nbytes": 4096,
2437
+ "byteOffset": 8388608
2438
+ },
2439
+ {
2440
+ "name": "gpt_neox.layers.15.mlp.dense_h_to_4h.bias",
2441
+ "shape": [
2442
+ 8192
2443
+ ],
2444
+ "dtype": "float32",
2445
+ "format": "f32-to-bf16",
2446
+ "nbytes": 16384,
2447
+ "byteOffset": 8392704
2448
+ },
2449
+ {
2450
+ "name": "gpt_neox.layers.15.mlp.dense_4h_to_h.bias",
2451
+ "shape": [
2452
+ 2048
2453
+ ],
2454
+ "dtype": "float32",
2455
+ "format": "f32-to-bf16",
2456
+ "nbytes": 4096,
2457
+ "byteOffset": 8409088
2458
+ },
2459
+ {
2460
+ "name": "gpt_neox.final_layer_norm.weight",
2461
+ "shape": [
2462
+ 2048
2463
+ ],
2464
+ "dtype": "float16",
2465
+ "format": "f32-to-bf16",
2466
+ "nbytes": 4096,
2467
+ "byteOffset": 8413184
2468
+ },
2469
+ {
2470
+ "name": "gpt_neox.final_layer_norm.bias",
2471
+ "shape": [
2472
+ 2048
2473
+ ],
2474
+ "dtype": "float16",
2475
+ "format": "f32-to-bf16",
2476
+ "nbytes": 4096,
2477
+ "byteOffset": 8417280
2478
+ }
2479
+ ],
2480
+ "md5sum": "4eaa42565d775e54b8a02497b2a7e15d"
2481
+ }
2482
+ ]
2483
+ }
params_shard_0.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2269985ea26f155f8db19232dd6cda9f4926d9ce8530fa8e6690caf5c17cb7de
3
+ size 206045184
params_shard_1.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e2dce5035ab0d2e8d18402f6a5e7349da9586404deb2fcf8f87a7c4b026e2fb0
3
+ size 25194496
params_shard_10.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:279aab2098eff72814e5fe9f8fc9c3d92d7ba5884bac7301013bf9ac79f70c83
3
+ size 25165824
params_shard_11.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e783a23a88fe880f427f5d49d9d3e6f6d12eb5da0f6a1812f9adf6a778e4824d
3
+ size 25325568
params_shard_12.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53e0831a9627f78615b2abfe430508c7fe62a2295405288d87a0d41cb4999ea8
3
+ size 33554432
params_shard_13.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90861636d430af5107b986efcc014f516f65b68b72ad08d76ca63448f3fdca78
3
+ size 33554432
params_shard_14.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:75897be780621c0592c4890fd5e6338d855d8b858625cce1d738083a1e99de60
3
+ size 25165824
params_shard_15.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d65adc4f6919ed11fde119d3f488f7f3de0d8edcfb80169678b2963ec39c1394
3
+ size 33554432
params_shard_16.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3c9d41413b979b8c2a06540966c2841dd0f77e8152374fed0f513871ca87078
3
+ size 33554432
params_shard_17.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53f8eeb2e037fb2e21189cf9e61eb7b0ba9143e86072264907a6e0ab1a727405
3
+ size 25165824
params_shard_18.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:019904efd16f067ff2f88ab0e007300277bf72f3358f55da56308dc0f756cf85
3
+ size 33554432
params_shard_19.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:07b606de1ce3624ee7c33e8d9a6587772dd506d6c26bae24e1ca76faddefca4b
3
+ size 33554432
params_shard_2.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d2aecc083beeb666775811d9c38f37c2c4717fe90bacb6929e88a9a1a18787b
3
+ size 33554432
params_shard_20.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb49ec6e934cdab2f0670cfe9c02215df07f41ab66812b1888b808710dd0b056
3
+ size 25165824
params_shard_21.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a4dea9f3409bc93ce83847aefa705a6079064701f6683c5623c23ffb324a0077
3
+ size 25325568
params_shard_22.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44b500a6eebff27cd561d39558d6bfe583d9d52df880c77c871a2898c54def1b
3
+ size 33554432
params_shard_23.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27d28611d1428c5bcb726705499d81a6608ad3157b32ec0e35b4884c5f733fbf
3
+ size 33554432
params_shard_24.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d758772c87c0519e07a60e9f31b2463087c9e2268a46bd368e588dc57699365
3
+ size 25165824
params_shard_25.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c90bf0b46b88c19fb03e0e630a10df57e7fa78d67255dbaabff8019fb0e86230
3
+ size 33554432
params_shard_26.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e2890a3bad1580421c80b5dae504d48fb1b3ad7e75a43c4060f4c94845fc6837
3
+ size 33554432
params_shard_27.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4997fd10e55968d21e4b98402300181caa877cc44d101aa4fbb5fe31abc79a7
3
+ size 25165824
params_shard_28.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ddf9a4139fc5c09375d678add5641a2c706c809ba3385a4c57f0fcc01bcbfb38
3
+ size 33554432
params_shard_29.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e421251c0851e5ca99edac0799000e988df2fef9d260189093b90752ab8090b1
3
+ size 33554432
params_shard_3.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a524a226c5de08f46890fb3948a0ed169f5737e36db22f3668078314bdcd6cb1
3
+ size 33554432
params_shard_30.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a2743d914cee00545f75c77ec86b6e8b0cd4fa55de9f1bf8637fd7f7f46f2ef7
3
+ size 25165824
params_shard_31.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a268eeabb6edef3d48b1b9f5b38af86b29cd17376b5f06b5a1acc28e13bdaf8
3
+ size 25325568
params_shard_32.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:07b6bb6d041966ee09dcecd3de3830c03428145395c0c08c3b1197d55c5b8535
3
+ size 33554432
params_shard_33.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05f77fea5b159bea6b8377f02e3bd75b05203ebb8c9017565153d11d7f960bc1
3
+ size 33554432
params_shard_34.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:22a4341dfaeff98b2da241a9780462a4af304b7a7dfd12617c03bc4ffc1628e0
3
+ size 25165824
params_shard_35.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:31c73be715fb47905e11996f4b1a3e080f2690c78e414dfec6476741b614c952
3
+ size 33554432
params_shard_36.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b1929fb29fa54335b791007da0eae330c0c528d6d5b4bd5636cd433e2fd4447a
3
+ size 33554432
params_shard_37.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:053d8e0f7e26b347f99994d7e6eae1e13fce26520126bb90a74ec231a2ffe409
3
+ size 25165824
params_shard_38.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1d3ce29249f4053dfd7f39eb81f618cc60780aa4d548293f025e9d614ed19e6
3
+ size 33554432
params_shard_39.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a00160950b8297ccf49d1c75dac810e0e8ca038dba717d56511e54133e4d6c6
3
+ size 33554432
params_shard_4.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41b8aab2717d84ddb59a370d015d232a539f714784be923de7a08f6e7e52e4aa
3
+ size 25165824
params_shard_40.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0471f983ca64bfb21c19b7ea86031a4debaf161101d96bc6f195afb1b7446db4
3
+ size 25165824
params_shard_41.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1aed8f8f831598017b90b0d614831cb6d550c839e63d74f91c73c0bb4e17e552
3
+ size 25325568
params_shard_42.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5767bc287ba5439a8d30d80e3b4d4634a5f1ce527a942b8ba7a3ddfb89f20937
3
+ size 33554432