Amai commited on
Commit
9476ea2
·
1 Parent(s): 306862b

Add weights

Browse files
EleutherAI_pythia-160m-android-q0f16.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6dabaf9c24b068305a170e47ed709a25c44a3ef349c33b2d853ad379f55e014d
3
+ size 306384
ir_dump/debug-final.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase0.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase1.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase2.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase3.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase4.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase5.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/kernel.cl ADDED
The diff for this file is too large to render. See raw diff
 
mlc-chat-config.json ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "0.1.0",
3
+ "model_type": "gpt_neox",
4
+ "quantization": "q0f16",
5
+ "model_config": {
6
+ "use_parallel_residual": true,
7
+ "hidden_size": 768,
8
+ "intermediate_size": 3072,
9
+ "num_attention_heads": 12,
10
+ "num_hidden_layers": 12,
11
+ "layer_norm_eps": 1e-05,
12
+ "vocab_size": 50304,
13
+ "rotary_pct": 0.25,
14
+ "position_embedding_base": 10000,
15
+ "context_window_size": 2048,
16
+ "head_dim": 64,
17
+ "prefill_chunk_size": 2048,
18
+ "tensor_parallel_shards": 1,
19
+ "ffn_out_dtype": "float32",
20
+ "max_batch_size": 128
21
+ },
22
+ "vocab_size": 50304,
23
+ "context_window_size": 2048,
24
+ "sliding_window_size": -1,
25
+ "prefill_chunk_size": 2048,
26
+ "attention_sink_size": -1,
27
+ "tensor_parallel_shards": 1,
28
+ "pipeline_parallel_stages": 1,
29
+ "temperature": 1.0,
30
+ "presence_penalty": 0.0,
31
+ "frequency_penalty": 0.0,
32
+ "repetition_penalty": 1.0,
33
+ "top_p": 1.0,
34
+ "tokenizer_files": [
35
+ "tokenizer.json",
36
+ "tokenizer_config.json"
37
+ ],
38
+ "tokenizer_info": {
39
+ "token_postproc_method": "byte_level",
40
+ "prepend_space_in_encode": false,
41
+ "strip_space_in_decode": false
42
+ },
43
+ "conv_template": {
44
+ "name": "gpt2",
45
+ "system_template": "{system_message}",
46
+ "system_message": "",
47
+ "system_prefix_token_ids": null,
48
+ "add_role_after_system_message": true,
49
+ "roles": {
50
+ "user": "",
51
+ "assistant": ""
52
+ },
53
+ "role_templates": {
54
+ "user": "{user_message}",
55
+ "assistant": "{assistant_message}",
56
+ "tool": "{tool_message}"
57
+ },
58
+ "messages": [],
59
+ "seps": [
60
+ ""
61
+ ],
62
+ "role_content_sep": "",
63
+ "role_empty_sep": "",
64
+ "stop_str": [
65
+ "</s>"
66
+ ],
67
+ "stop_token_ids": [
68
+ 50256
69
+ ],
70
+ "function_string": "",
71
+ "use_function_calling": false
72
+ },
73
+ "pad_token_id": 0,
74
+ "bos_token_id": 0,
75
+ "eos_token_id": 0
76
+ }
ndarray-cache-b16.json ADDED
@@ -0,0 +1,1603 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "ParamSize": 148,
4
+ "ParamBytes": 324738048.0,
5
+ "BitsPerParam": 16.00454205660538
6
+ },
7
+ "records": [
8
+ {
9
+ "dataPath": "params_shard_0.bin",
10
+ "format": "raw-shard",
11
+ "nbytes": 77266944,
12
+ "records": [
13
+ {
14
+ "name": "gpt_neox.embed_in.weight",
15
+ "shape": [
16
+ 50304,
17
+ 768
18
+ ],
19
+ "dtype": "float16",
20
+ "format": "f32-to-bf16",
21
+ "nbytes": 77266944,
22
+ "byteOffset": 0
23
+ }
24
+ ],
25
+ "md5sum": "2481121002c12eed2951b729fa17b644"
26
+ },
27
+ {
28
+ "dataPath": "params_shard_1.bin",
29
+ "format": "raw-shard",
30
+ "nbytes": 33082368,
31
+ "records": [
32
+ {
33
+ "name": "gpt_neox.layers.0.input_layernorm.weight",
34
+ "shape": [
35
+ 768
36
+ ],
37
+ "dtype": "float16",
38
+ "format": "f32-to-bf16",
39
+ "nbytes": 1536,
40
+ "byteOffset": 0
41
+ },
42
+ {
43
+ "name": "gpt_neox.layers.0.input_layernorm.bias",
44
+ "shape": [
45
+ 768
46
+ ],
47
+ "dtype": "float16",
48
+ "format": "f32-to-bf16",
49
+ "nbytes": 1536,
50
+ "byteOffset": 1536
51
+ },
52
+ {
53
+ "name": "gpt_neox.layers.0.post_attention_layernorm.weight",
54
+ "shape": [
55
+ 768
56
+ ],
57
+ "dtype": "float16",
58
+ "format": "f32-to-bf16",
59
+ "nbytes": 1536,
60
+ "byteOffset": 3072
61
+ },
62
+ {
63
+ "name": "gpt_neox.layers.0.post_attention_layernorm.bias",
64
+ "shape": [
65
+ 768
66
+ ],
67
+ "dtype": "float16",
68
+ "format": "f32-to-bf16",
69
+ "nbytes": 1536,
70
+ "byteOffset": 4608
71
+ },
72
+ {
73
+ "name": "gpt_neox.layers.0.attention.query_key_value.weight",
74
+ "shape": [
75
+ 2304,
76
+ 768
77
+ ],
78
+ "dtype": "float16",
79
+ "format": "f32-to-bf16",
80
+ "nbytes": 3538944,
81
+ "byteOffset": 6144
82
+ },
83
+ {
84
+ "name": "gpt_neox.layers.0.attention.query_key_value.bias",
85
+ "shape": [
86
+ 2304
87
+ ],
88
+ "dtype": "float16",
89
+ "format": "f32-to-bf16",
90
+ "nbytes": 4608,
91
+ "byteOffset": 3545088
92
+ },
93
+ {
94
+ "name": "gpt_neox.layers.0.attention.dense.weight",
95
+ "shape": [
96
+ 768,
97
+ 768
98
+ ],
99
+ "dtype": "float16",
100
+ "format": "f32-to-bf16",
101
+ "nbytes": 1179648,
102
+ "byteOffset": 3549696
103
+ },
104
+ {
105
+ "name": "gpt_neox.layers.0.attention.dense.bias",
106
+ "shape": [
107
+ 768
108
+ ],
109
+ "dtype": "float16",
110
+ "format": "f32-to-bf16",
111
+ "nbytes": 1536,
112
+ "byteOffset": 4729344
113
+ },
114
+ {
115
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight",
116
+ "shape": [
117
+ 3072,
118
+ 768
119
+ ],
120
+ "dtype": "float16",
121
+ "format": "f32-to-bf16",
122
+ "nbytes": 4718592,
123
+ "byteOffset": 4730880
124
+ },
125
+ {
126
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
127
+ "shape": [
128
+ 3072
129
+ ],
130
+ "dtype": "bfloat16",
131
+ "format": "raw",
132
+ "nbytes": 6144,
133
+ "byteOffset": 9449472
134
+ },
135
+ {
136
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight",
137
+ "shape": [
138
+ 768,
139
+ 3072
140
+ ],
141
+ "dtype": "float16",
142
+ "format": "f32-to-bf16",
143
+ "nbytes": 4718592,
144
+ "byteOffset": 9455616
145
+ },
146
+ {
147
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
148
+ "shape": [
149
+ 768
150
+ ],
151
+ "dtype": "bfloat16",
152
+ "format": "raw",
153
+ "nbytes": 1536,
154
+ "byteOffset": 14174208
155
+ },
156
+ {
157
+ "name": "gpt_neox.layers.1.input_layernorm.weight",
158
+ "shape": [
159
+ 768
160
+ ],
161
+ "dtype": "float16",
162
+ "format": "f32-to-bf16",
163
+ "nbytes": 1536,
164
+ "byteOffset": 14175744
165
+ },
166
+ {
167
+ "name": "gpt_neox.layers.1.input_layernorm.bias",
168
+ "shape": [
169
+ 768
170
+ ],
171
+ "dtype": "float16",
172
+ "format": "f32-to-bf16",
173
+ "nbytes": 1536,
174
+ "byteOffset": 14177280
175
+ },
176
+ {
177
+ "name": "gpt_neox.layers.1.post_attention_layernorm.weight",
178
+ "shape": [
179
+ 768
180
+ ],
181
+ "dtype": "float16",
182
+ "format": "f32-to-bf16",
183
+ "nbytes": 1536,
184
+ "byteOffset": 14178816
185
+ },
186
+ {
187
+ "name": "gpt_neox.layers.1.post_attention_layernorm.bias",
188
+ "shape": [
189
+ 768
190
+ ],
191
+ "dtype": "float16",
192
+ "format": "f32-to-bf16",
193
+ "nbytes": 1536,
194
+ "byteOffset": 14180352
195
+ },
196
+ {
197
+ "name": "gpt_neox.layers.1.attention.query_key_value.weight",
198
+ "shape": [
199
+ 2304,
200
+ 768
201
+ ],
202
+ "dtype": "float16",
203
+ "format": "f32-to-bf16",
204
+ "nbytes": 3538944,
205
+ "byteOffset": 14181888
206
+ },
207
+ {
208
+ "name": "gpt_neox.layers.1.attention.query_key_value.bias",
209
+ "shape": [
210
+ 2304
211
+ ],
212
+ "dtype": "float16",
213
+ "format": "f32-to-bf16",
214
+ "nbytes": 4608,
215
+ "byteOffset": 17720832
216
+ },
217
+ {
218
+ "name": "gpt_neox.layers.1.attention.dense.weight",
219
+ "shape": [
220
+ 768,
221
+ 768
222
+ ],
223
+ "dtype": "float16",
224
+ "format": "f32-to-bf16",
225
+ "nbytes": 1179648,
226
+ "byteOffset": 17725440
227
+ },
228
+ {
229
+ "name": "gpt_neox.layers.1.attention.dense.bias",
230
+ "shape": [
231
+ 768
232
+ ],
233
+ "dtype": "float16",
234
+ "format": "f32-to-bf16",
235
+ "nbytes": 1536,
236
+ "byteOffset": 18905088
237
+ },
238
+ {
239
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight",
240
+ "shape": [
241
+ 3072,
242
+ 768
243
+ ],
244
+ "dtype": "float16",
245
+ "format": "f32-to-bf16",
246
+ "nbytes": 4718592,
247
+ "byteOffset": 18906624
248
+ },
249
+ {
250
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
251
+ "shape": [
252
+ 3072
253
+ ],
254
+ "dtype": "bfloat16",
255
+ "format": "raw",
256
+ "nbytes": 6144,
257
+ "byteOffset": 23625216
258
+ },
259
+ {
260
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight",
261
+ "shape": [
262
+ 768,
263
+ 3072
264
+ ],
265
+ "dtype": "float16",
266
+ "format": "f32-to-bf16",
267
+ "nbytes": 4718592,
268
+ "byteOffset": 23631360
269
+ },
270
+ {
271
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
272
+ "shape": [
273
+ 768
274
+ ],
275
+ "dtype": "bfloat16",
276
+ "format": "raw",
277
+ "nbytes": 1536,
278
+ "byteOffset": 28349952
279
+ },
280
+ {
281
+ "name": "gpt_neox.layers.2.input_layernorm.weight",
282
+ "shape": [
283
+ 768
284
+ ],
285
+ "dtype": "float16",
286
+ "format": "f32-to-bf16",
287
+ "nbytes": 1536,
288
+ "byteOffset": 28351488
289
+ },
290
+ {
291
+ "name": "gpt_neox.layers.2.input_layernorm.bias",
292
+ "shape": [
293
+ 768
294
+ ],
295
+ "dtype": "float16",
296
+ "format": "f32-to-bf16",
297
+ "nbytes": 1536,
298
+ "byteOffset": 28353024
299
+ },
300
+ {
301
+ "name": "gpt_neox.layers.2.post_attention_layernorm.weight",
302
+ "shape": [
303
+ 768
304
+ ],
305
+ "dtype": "float16",
306
+ "format": "f32-to-bf16",
307
+ "nbytes": 1536,
308
+ "byteOffset": 28354560
309
+ },
310
+ {
311
+ "name": "gpt_neox.layers.2.post_attention_layernorm.bias",
312
+ "shape": [
313
+ 768
314
+ ],
315
+ "dtype": "float16",
316
+ "format": "f32-to-bf16",
317
+ "nbytes": 1536,
318
+ "byteOffset": 28356096
319
+ },
320
+ {
321
+ "name": "gpt_neox.layers.2.attention.query_key_value.weight",
322
+ "shape": [
323
+ 2304,
324
+ 768
325
+ ],
326
+ "dtype": "float16",
327
+ "format": "f32-to-bf16",
328
+ "nbytes": 3538944,
329
+ "byteOffset": 28357632
330
+ },
331
+ {
332
+ "name": "gpt_neox.layers.2.attention.query_key_value.bias",
333
+ "shape": [
334
+ 2304
335
+ ],
336
+ "dtype": "float16",
337
+ "format": "f32-to-bf16",
338
+ "nbytes": 4608,
339
+ "byteOffset": 31896576
340
+ },
341
+ {
342
+ "name": "gpt_neox.layers.2.attention.dense.weight",
343
+ "shape": [
344
+ 768,
345
+ 768
346
+ ],
347
+ "dtype": "float16",
348
+ "format": "f32-to-bf16",
349
+ "nbytes": 1179648,
350
+ "byteOffset": 31901184
351
+ },
352
+ {
353
+ "name": "gpt_neox.layers.2.attention.dense.bias",
354
+ "shape": [
355
+ 768
356
+ ],
357
+ "dtype": "float16",
358
+ "format": "f32-to-bf16",
359
+ "nbytes": 1536,
360
+ "byteOffset": 33080832
361
+ }
362
+ ],
363
+ "md5sum": "f152a892aaec5607fd2b89419435780a"
364
+ },
365
+ {
366
+ "dataPath": "params_shard_2.bin",
367
+ "format": "raw-shard",
368
+ "nbytes": 33076224,
369
+ "records": [
370
+ {
371
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight",
372
+ "shape": [
373
+ 3072,
374
+ 768
375
+ ],
376
+ "dtype": "float16",
377
+ "format": "f32-to-bf16",
378
+ "nbytes": 4718592,
379
+ "byteOffset": 0
380
+ },
381
+ {
382
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
383
+ "shape": [
384
+ 3072
385
+ ],
386
+ "dtype": "bfloat16",
387
+ "format": "raw",
388
+ "nbytes": 6144,
389
+ "byteOffset": 4718592
390
+ },
391
+ {
392
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight",
393
+ "shape": [
394
+ 768,
395
+ 3072
396
+ ],
397
+ "dtype": "float16",
398
+ "format": "f32-to-bf16",
399
+ "nbytes": 4718592,
400
+ "byteOffset": 4724736
401
+ },
402
+ {
403
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
404
+ "shape": [
405
+ 768
406
+ ],
407
+ "dtype": "bfloat16",
408
+ "format": "raw",
409
+ "nbytes": 1536,
410
+ "byteOffset": 9443328
411
+ },
412
+ {
413
+ "name": "gpt_neox.layers.3.input_layernorm.weight",
414
+ "shape": [
415
+ 768
416
+ ],
417
+ "dtype": "float16",
418
+ "format": "f32-to-bf16",
419
+ "nbytes": 1536,
420
+ "byteOffset": 9444864
421
+ },
422
+ {
423
+ "name": "gpt_neox.layers.3.input_layernorm.bias",
424
+ "shape": [
425
+ 768
426
+ ],
427
+ "dtype": "float16",
428
+ "format": "f32-to-bf16",
429
+ "nbytes": 1536,
430
+ "byteOffset": 9446400
431
+ },
432
+ {
433
+ "name": "gpt_neox.layers.3.post_attention_layernorm.weight",
434
+ "shape": [
435
+ 768
436
+ ],
437
+ "dtype": "float16",
438
+ "format": "f32-to-bf16",
439
+ "nbytes": 1536,
440
+ "byteOffset": 9447936
441
+ },
442
+ {
443
+ "name": "gpt_neox.layers.3.post_attention_layernorm.bias",
444
+ "shape": [
445
+ 768
446
+ ],
447
+ "dtype": "float16",
448
+ "format": "f32-to-bf16",
449
+ "nbytes": 1536,
450
+ "byteOffset": 9449472
451
+ },
452
+ {
453
+ "name": "gpt_neox.layers.3.attention.query_key_value.weight",
454
+ "shape": [
455
+ 2304,
456
+ 768
457
+ ],
458
+ "dtype": "float16",
459
+ "format": "f32-to-bf16",
460
+ "nbytes": 3538944,
461
+ "byteOffset": 9451008
462
+ },
463
+ {
464
+ "name": "gpt_neox.layers.3.attention.query_key_value.bias",
465
+ "shape": [
466
+ 2304
467
+ ],
468
+ "dtype": "float16",
469
+ "format": "f32-to-bf16",
470
+ "nbytes": 4608,
471
+ "byteOffset": 12989952
472
+ },
473
+ {
474
+ "name": "gpt_neox.layers.3.attention.dense.weight",
475
+ "shape": [
476
+ 768,
477
+ 768
478
+ ],
479
+ "dtype": "float16",
480
+ "format": "f32-to-bf16",
481
+ "nbytes": 1179648,
482
+ "byteOffset": 12994560
483
+ },
484
+ {
485
+ "name": "gpt_neox.layers.3.attention.dense.bias",
486
+ "shape": [
487
+ 768
488
+ ],
489
+ "dtype": "float16",
490
+ "format": "f32-to-bf16",
491
+ "nbytes": 1536,
492
+ "byteOffset": 14174208
493
+ },
494
+ {
495
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight",
496
+ "shape": [
497
+ 3072,
498
+ 768
499
+ ],
500
+ "dtype": "float16",
501
+ "format": "f32-to-bf16",
502
+ "nbytes": 4718592,
503
+ "byteOffset": 14175744
504
+ },
505
+ {
506
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
507
+ "shape": [
508
+ 3072
509
+ ],
510
+ "dtype": "bfloat16",
511
+ "format": "raw",
512
+ "nbytes": 6144,
513
+ "byteOffset": 18894336
514
+ },
515
+ {
516
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight",
517
+ "shape": [
518
+ 768,
519
+ 3072
520
+ ],
521
+ "dtype": "float16",
522
+ "format": "f32-to-bf16",
523
+ "nbytes": 4718592,
524
+ "byteOffset": 18900480
525
+ },
526
+ {
527
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
528
+ "shape": [
529
+ 768
530
+ ],
531
+ "dtype": "bfloat16",
532
+ "format": "raw",
533
+ "nbytes": 1536,
534
+ "byteOffset": 23619072
535
+ },
536
+ {
537
+ "name": "gpt_neox.layers.4.input_layernorm.weight",
538
+ "shape": [
539
+ 768
540
+ ],
541
+ "dtype": "float16",
542
+ "format": "f32-to-bf16",
543
+ "nbytes": 1536,
544
+ "byteOffset": 23620608
545
+ },
546
+ {
547
+ "name": "gpt_neox.layers.4.input_layernorm.bias",
548
+ "shape": [
549
+ 768
550
+ ],
551
+ "dtype": "float16",
552
+ "format": "f32-to-bf16",
553
+ "nbytes": 1536,
554
+ "byteOffset": 23622144
555
+ },
556
+ {
557
+ "name": "gpt_neox.layers.4.post_attention_layernorm.weight",
558
+ "shape": [
559
+ 768
560
+ ],
561
+ "dtype": "float16",
562
+ "format": "f32-to-bf16",
563
+ "nbytes": 1536,
564
+ "byteOffset": 23623680
565
+ },
566
+ {
567
+ "name": "gpt_neox.layers.4.post_attention_layernorm.bias",
568
+ "shape": [
569
+ 768
570
+ ],
571
+ "dtype": "float16",
572
+ "format": "f32-to-bf16",
573
+ "nbytes": 1536,
574
+ "byteOffset": 23625216
575
+ },
576
+ {
577
+ "name": "gpt_neox.layers.4.attention.query_key_value.weight",
578
+ "shape": [
579
+ 2304,
580
+ 768
581
+ ],
582
+ "dtype": "float16",
583
+ "format": "f32-to-bf16",
584
+ "nbytes": 3538944,
585
+ "byteOffset": 23626752
586
+ },
587
+ {
588
+ "name": "gpt_neox.layers.4.attention.query_key_value.bias",
589
+ "shape": [
590
+ 2304
591
+ ],
592
+ "dtype": "float16",
593
+ "format": "f32-to-bf16",
594
+ "nbytes": 4608,
595
+ "byteOffset": 27165696
596
+ },
597
+ {
598
+ "name": "gpt_neox.layers.4.attention.dense.weight",
599
+ "shape": [
600
+ 768,
601
+ 768
602
+ ],
603
+ "dtype": "float16",
604
+ "format": "f32-to-bf16",
605
+ "nbytes": 1179648,
606
+ "byteOffset": 27170304
607
+ },
608
+ {
609
+ "name": "gpt_neox.layers.4.attention.dense.bias",
610
+ "shape": [
611
+ 768
612
+ ],
613
+ "dtype": "float16",
614
+ "format": "f32-to-bf16",
615
+ "nbytes": 1536,
616
+ "byteOffset": 28349952
617
+ },
618
+ {
619
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight",
620
+ "shape": [
621
+ 3072,
622
+ 768
623
+ ],
624
+ "dtype": "float16",
625
+ "format": "f32-to-bf16",
626
+ "nbytes": 4718592,
627
+ "byteOffset": 28351488
628
+ },
629
+ {
630
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
631
+ "shape": [
632
+ 3072
633
+ ],
634
+ "dtype": "bfloat16",
635
+ "format": "raw",
636
+ "nbytes": 6144,
637
+ "byteOffset": 33070080
638
+ }
639
+ ],
640
+ "md5sum": "57ece0af7da914417b0f9080c0bdac26"
641
+ },
642
+ {
643
+ "dataPath": "params_shard_3.bin",
644
+ "format": "raw-shard",
645
+ "nbytes": 33077760,
646
+ "records": [
647
+ {
648
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight",
649
+ "shape": [
650
+ 768,
651
+ 3072
652
+ ],
653
+ "dtype": "float16",
654
+ "format": "f32-to-bf16",
655
+ "nbytes": 4718592,
656
+ "byteOffset": 0
657
+ },
658
+ {
659
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
660
+ "shape": [
661
+ 768
662
+ ],
663
+ "dtype": "bfloat16",
664
+ "format": "raw",
665
+ "nbytes": 1536,
666
+ "byteOffset": 4718592
667
+ },
668
+ {
669
+ "name": "gpt_neox.layers.5.input_layernorm.weight",
670
+ "shape": [
671
+ 768
672
+ ],
673
+ "dtype": "float16",
674
+ "format": "f32-to-bf16",
675
+ "nbytes": 1536,
676
+ "byteOffset": 4720128
677
+ },
678
+ {
679
+ "name": "gpt_neox.layers.5.input_layernorm.bias",
680
+ "shape": [
681
+ 768
682
+ ],
683
+ "dtype": "float16",
684
+ "format": "f32-to-bf16",
685
+ "nbytes": 1536,
686
+ "byteOffset": 4721664
687
+ },
688
+ {
689
+ "name": "gpt_neox.layers.5.post_attention_layernorm.weight",
690
+ "shape": [
691
+ 768
692
+ ],
693
+ "dtype": "float16",
694
+ "format": "f32-to-bf16",
695
+ "nbytes": 1536,
696
+ "byteOffset": 4723200
697
+ },
698
+ {
699
+ "name": "gpt_neox.layers.5.post_attention_layernorm.bias",
700
+ "shape": [
701
+ 768
702
+ ],
703
+ "dtype": "float16",
704
+ "format": "f32-to-bf16",
705
+ "nbytes": 1536,
706
+ "byteOffset": 4724736
707
+ },
708
+ {
709
+ "name": "gpt_neox.layers.5.attention.query_key_value.weight",
710
+ "shape": [
711
+ 2304,
712
+ 768
713
+ ],
714
+ "dtype": "float16",
715
+ "format": "f32-to-bf16",
716
+ "nbytes": 3538944,
717
+ "byteOffset": 4726272
718
+ },
719
+ {
720
+ "name": "gpt_neox.layers.5.attention.query_key_value.bias",
721
+ "shape": [
722
+ 2304
723
+ ],
724
+ "dtype": "float16",
725
+ "format": "f32-to-bf16",
726
+ "nbytes": 4608,
727
+ "byteOffset": 8265216
728
+ },
729
+ {
730
+ "name": "gpt_neox.layers.5.attention.dense.weight",
731
+ "shape": [
732
+ 768,
733
+ 768
734
+ ],
735
+ "dtype": "float16",
736
+ "format": "f32-to-bf16",
737
+ "nbytes": 1179648,
738
+ "byteOffset": 8269824
739
+ },
740
+ {
741
+ "name": "gpt_neox.layers.5.attention.dense.bias",
742
+ "shape": [
743
+ 768
744
+ ],
745
+ "dtype": "float16",
746
+ "format": "f32-to-bf16",
747
+ "nbytes": 1536,
748
+ "byteOffset": 9449472
749
+ },
750
+ {
751
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight",
752
+ "shape": [
753
+ 3072,
754
+ 768
755
+ ],
756
+ "dtype": "float16",
757
+ "format": "f32-to-bf16",
758
+ "nbytes": 4718592,
759
+ "byteOffset": 9451008
760
+ },
761
+ {
762
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
763
+ "shape": [
764
+ 3072
765
+ ],
766
+ "dtype": "bfloat16",
767
+ "format": "raw",
768
+ "nbytes": 6144,
769
+ "byteOffset": 14169600
770
+ },
771
+ {
772
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight",
773
+ "shape": [
774
+ 768,
775
+ 3072
776
+ ],
777
+ "dtype": "float16",
778
+ "format": "f32-to-bf16",
779
+ "nbytes": 4718592,
780
+ "byteOffset": 14175744
781
+ },
782
+ {
783
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
784
+ "shape": [
785
+ 768
786
+ ],
787
+ "dtype": "bfloat16",
788
+ "format": "raw",
789
+ "nbytes": 1536,
790
+ "byteOffset": 18894336
791
+ },
792
+ {
793
+ "name": "gpt_neox.layers.6.input_layernorm.weight",
794
+ "shape": [
795
+ 768
796
+ ],
797
+ "dtype": "float16",
798
+ "format": "f32-to-bf16",
799
+ "nbytes": 1536,
800
+ "byteOffset": 18895872
801
+ },
802
+ {
803
+ "name": "gpt_neox.layers.6.input_layernorm.bias",
804
+ "shape": [
805
+ 768
806
+ ],
807
+ "dtype": "float16",
808
+ "format": "f32-to-bf16",
809
+ "nbytes": 1536,
810
+ "byteOffset": 18897408
811
+ },
812
+ {
813
+ "name": "gpt_neox.layers.6.post_attention_layernorm.weight",
814
+ "shape": [
815
+ 768
816
+ ],
817
+ "dtype": "float16",
818
+ "format": "f32-to-bf16",
819
+ "nbytes": 1536,
820
+ "byteOffset": 18898944
821
+ },
822
+ {
823
+ "name": "gpt_neox.layers.6.post_attention_layernorm.bias",
824
+ "shape": [
825
+ 768
826
+ ],
827
+ "dtype": "float16",
828
+ "format": "f32-to-bf16",
829
+ "nbytes": 1536,
830
+ "byteOffset": 18900480
831
+ },
832
+ {
833
+ "name": "gpt_neox.layers.6.attention.query_key_value.weight",
834
+ "shape": [
835
+ 2304,
836
+ 768
837
+ ],
838
+ "dtype": "float16",
839
+ "format": "f32-to-bf16",
840
+ "nbytes": 3538944,
841
+ "byteOffset": 18902016
842
+ },
843
+ {
844
+ "name": "gpt_neox.layers.6.attention.query_key_value.bias",
845
+ "shape": [
846
+ 2304
847
+ ],
848
+ "dtype": "float16",
849
+ "format": "f32-to-bf16",
850
+ "nbytes": 4608,
851
+ "byteOffset": 22440960
852
+ },
853
+ {
854
+ "name": "gpt_neox.layers.6.attention.dense.weight",
855
+ "shape": [
856
+ 768,
857
+ 768
858
+ ],
859
+ "dtype": "float16",
860
+ "format": "f32-to-bf16",
861
+ "nbytes": 1179648,
862
+ "byteOffset": 22445568
863
+ },
864
+ {
865
+ "name": "gpt_neox.layers.6.attention.dense.bias",
866
+ "shape": [
867
+ 768
868
+ ],
869
+ "dtype": "float16",
870
+ "format": "f32-to-bf16",
871
+ "nbytes": 1536,
872
+ "byteOffset": 23625216
873
+ },
874
+ {
875
+ "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.weight",
876
+ "shape": [
877
+ 3072,
878
+ 768
879
+ ],
880
+ "dtype": "float16",
881
+ "format": "f32-to-bf16",
882
+ "nbytes": 4718592,
883
+ "byteOffset": 23626752
884
+ },
885
+ {
886
+ "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
887
+ "shape": [
888
+ 3072
889
+ ],
890
+ "dtype": "bfloat16",
891
+ "format": "raw",
892
+ "nbytes": 6144,
893
+ "byteOffset": 28345344
894
+ },
895
+ {
896
+ "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.weight",
897
+ "shape": [
898
+ 768,
899
+ 3072
900
+ ],
901
+ "dtype": "float16",
902
+ "format": "f32-to-bf16",
903
+ "nbytes": 4718592,
904
+ "byteOffset": 28351488
905
+ },
906
+ {
907
+ "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
908
+ "shape": [
909
+ 768
910
+ ],
911
+ "dtype": "bfloat16",
912
+ "format": "raw",
913
+ "nbytes": 1536,
914
+ "byteOffset": 33070080
915
+ },
916
+ {
917
+ "name": "gpt_neox.layers.7.input_layernorm.weight",
918
+ "shape": [
919
+ 768
920
+ ],
921
+ "dtype": "float16",
922
+ "format": "f32-to-bf16",
923
+ "nbytes": 1536,
924
+ "byteOffset": 33071616
925
+ },
926
+ {
927
+ "name": "gpt_neox.layers.7.input_layernorm.bias",
928
+ "shape": [
929
+ 768
930
+ ],
931
+ "dtype": "float16",
932
+ "format": "f32-to-bf16",
933
+ "nbytes": 1536,
934
+ "byteOffset": 33073152
935
+ },
936
+ {
937
+ "name": "gpt_neox.layers.7.post_attention_layernorm.weight",
938
+ "shape": [
939
+ 768
940
+ ],
941
+ "dtype": "float16",
942
+ "format": "f32-to-bf16",
943
+ "nbytes": 1536,
944
+ "byteOffset": 33074688
945
+ },
946
+ {
947
+ "name": "gpt_neox.layers.7.post_attention_layernorm.bias",
948
+ "shape": [
949
+ 768
950
+ ],
951
+ "dtype": "float16",
952
+ "format": "f32-to-bf16",
953
+ "nbytes": 1536,
954
+ "byteOffset": 33076224
955
+ }
956
+ ],
957
+ "md5sum": "773204e0e5615a84b4109b95d21abb1e"
958
+ },
959
+ {
960
+ "dataPath": "params_shard_4.bin",
961
+ "format": "raw-shard",
962
+ "nbytes": 33076224,
963
+ "records": [
964
+ {
965
+ "name": "gpt_neox.layers.7.attention.query_key_value.weight",
966
+ "shape": [
967
+ 2304,
968
+ 768
969
+ ],
970
+ "dtype": "float16",
971
+ "format": "f32-to-bf16",
972
+ "nbytes": 3538944,
973
+ "byteOffset": 0
974
+ },
975
+ {
976
+ "name": "gpt_neox.layers.7.attention.query_key_value.bias",
977
+ "shape": [
978
+ 2304
979
+ ],
980
+ "dtype": "float16",
981
+ "format": "f32-to-bf16",
982
+ "nbytes": 4608,
983
+ "byteOffset": 3538944
984
+ },
985
+ {
986
+ "name": "gpt_neox.layers.7.attention.dense.weight",
987
+ "shape": [
988
+ 768,
989
+ 768
990
+ ],
991
+ "dtype": "float16",
992
+ "format": "f32-to-bf16",
993
+ "nbytes": 1179648,
994
+ "byteOffset": 3543552
995
+ },
996
+ {
997
+ "name": "gpt_neox.layers.7.attention.dense.bias",
998
+ "shape": [
999
+ 768
1000
+ ],
1001
+ "dtype": "float16",
1002
+ "format": "f32-to-bf16",
1003
+ "nbytes": 1536,
1004
+ "byteOffset": 4723200
1005
+ },
1006
+ {
1007
+ "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.weight",
1008
+ "shape": [
1009
+ 3072,
1010
+ 768
1011
+ ],
1012
+ "dtype": "float16",
1013
+ "format": "f32-to-bf16",
1014
+ "nbytes": 4718592,
1015
+ "byteOffset": 4724736
1016
+ },
1017
+ {
1018
+ "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
1019
+ "shape": [
1020
+ 3072
1021
+ ],
1022
+ "dtype": "bfloat16",
1023
+ "format": "raw",
1024
+ "nbytes": 6144,
1025
+ "byteOffset": 9443328
1026
+ },
1027
+ {
1028
+ "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.weight",
1029
+ "shape": [
1030
+ 768,
1031
+ 3072
1032
+ ],
1033
+ "dtype": "float16",
1034
+ "format": "f32-to-bf16",
1035
+ "nbytes": 4718592,
1036
+ "byteOffset": 9449472
1037
+ },
1038
+ {
1039
+ "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
1040
+ "shape": [
1041
+ 768
1042
+ ],
1043
+ "dtype": "bfloat16",
1044
+ "format": "raw",
1045
+ "nbytes": 1536,
1046
+ "byteOffset": 14168064
1047
+ },
1048
+ {
1049
+ "name": "gpt_neox.layers.8.input_layernorm.weight",
1050
+ "shape": [
1051
+ 768
1052
+ ],
1053
+ "dtype": "float16",
1054
+ "format": "f32-to-bf16",
1055
+ "nbytes": 1536,
1056
+ "byteOffset": 14169600
1057
+ },
1058
+ {
1059
+ "name": "gpt_neox.layers.8.input_layernorm.bias",
1060
+ "shape": [
1061
+ 768
1062
+ ],
1063
+ "dtype": "float16",
1064
+ "format": "f32-to-bf16",
1065
+ "nbytes": 1536,
1066
+ "byteOffset": 14171136
1067
+ },
1068
+ {
1069
+ "name": "gpt_neox.layers.8.post_attention_layernorm.weight",
1070
+ "shape": [
1071
+ 768
1072
+ ],
1073
+ "dtype": "float16",
1074
+ "format": "f32-to-bf16",
1075
+ "nbytes": 1536,
1076
+ "byteOffset": 14172672
1077
+ },
1078
+ {
1079
+ "name": "gpt_neox.layers.8.post_attention_layernorm.bias",
1080
+ "shape": [
1081
+ 768
1082
+ ],
1083
+ "dtype": "float16",
1084
+ "format": "f32-to-bf16",
1085
+ "nbytes": 1536,
1086
+ "byteOffset": 14174208
1087
+ },
1088
+ {
1089
+ "name": "gpt_neox.layers.8.attention.query_key_value.weight",
1090
+ "shape": [
1091
+ 2304,
1092
+ 768
1093
+ ],
1094
+ "dtype": "float16",
1095
+ "format": "f32-to-bf16",
1096
+ "nbytes": 3538944,
1097
+ "byteOffset": 14175744
1098
+ },
1099
+ {
1100
+ "name": "gpt_neox.layers.8.attention.query_key_value.bias",
1101
+ "shape": [
1102
+ 2304
1103
+ ],
1104
+ "dtype": "float16",
1105
+ "format": "f32-to-bf16",
1106
+ "nbytes": 4608,
1107
+ "byteOffset": 17714688
1108
+ },
1109
+ {
1110
+ "name": "gpt_neox.layers.8.attention.dense.weight",
1111
+ "shape": [
1112
+ 768,
1113
+ 768
1114
+ ],
1115
+ "dtype": "float16",
1116
+ "format": "f32-to-bf16",
1117
+ "nbytes": 1179648,
1118
+ "byteOffset": 17719296
1119
+ },
1120
+ {
1121
+ "name": "gpt_neox.layers.8.attention.dense.bias",
1122
+ "shape": [
1123
+ 768
1124
+ ],
1125
+ "dtype": "float16",
1126
+ "format": "f32-to-bf16",
1127
+ "nbytes": 1536,
1128
+ "byteOffset": 18898944
1129
+ },
1130
+ {
1131
+ "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.weight",
1132
+ "shape": [
1133
+ 3072,
1134
+ 768
1135
+ ],
1136
+ "dtype": "float16",
1137
+ "format": "f32-to-bf16",
1138
+ "nbytes": 4718592,
1139
+ "byteOffset": 18900480
1140
+ },
1141
+ {
1142
+ "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
1143
+ "shape": [
1144
+ 3072
1145
+ ],
1146
+ "dtype": "bfloat16",
1147
+ "format": "raw",
1148
+ "nbytes": 6144,
1149
+ "byteOffset": 23619072
1150
+ },
1151
+ {
1152
+ "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.weight",
1153
+ "shape": [
1154
+ 768,
1155
+ 3072
1156
+ ],
1157
+ "dtype": "float16",
1158
+ "format": "f32-to-bf16",
1159
+ "nbytes": 4718592,
1160
+ "byteOffset": 23625216
1161
+ },
1162
+ {
1163
+ "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
1164
+ "shape": [
1165
+ 768
1166
+ ],
1167
+ "dtype": "bfloat16",
1168
+ "format": "raw",
1169
+ "nbytes": 1536,
1170
+ "byteOffset": 28343808
1171
+ },
1172
+ {
1173
+ "name": "gpt_neox.layers.9.input_layernorm.weight",
1174
+ "shape": [
1175
+ 768
1176
+ ],
1177
+ "dtype": "float16",
1178
+ "format": "f32-to-bf16",
1179
+ "nbytes": 1536,
1180
+ "byteOffset": 28345344
1181
+ },
1182
+ {
1183
+ "name": "gpt_neox.layers.9.input_layernorm.bias",
1184
+ "shape": [
1185
+ 768
1186
+ ],
1187
+ "dtype": "float16",
1188
+ "format": "f32-to-bf16",
1189
+ "nbytes": 1536,
1190
+ "byteOffset": 28346880
1191
+ },
1192
+ {
1193
+ "name": "gpt_neox.layers.9.post_attention_layernorm.weight",
1194
+ "shape": [
1195
+ 768
1196
+ ],
1197
+ "dtype": "float16",
1198
+ "format": "f32-to-bf16",
1199
+ "nbytes": 1536,
1200
+ "byteOffset": 28348416
1201
+ },
1202
+ {
1203
+ "name": "gpt_neox.layers.9.post_attention_layernorm.bias",
1204
+ "shape": [
1205
+ 768
1206
+ ],
1207
+ "dtype": "float16",
1208
+ "format": "f32-to-bf16",
1209
+ "nbytes": 1536,
1210
+ "byteOffset": 28349952
1211
+ },
1212
+ {
1213
+ "name": "gpt_neox.layers.9.attention.query_key_value.weight",
1214
+ "shape": [
1215
+ 2304,
1216
+ 768
1217
+ ],
1218
+ "dtype": "float16",
1219
+ "format": "f32-to-bf16",
1220
+ "nbytes": 3538944,
1221
+ "byteOffset": 28351488
1222
+ },
1223
+ {
1224
+ "name": "gpt_neox.layers.9.attention.query_key_value.bias",
1225
+ "shape": [
1226
+ 2304
1227
+ ],
1228
+ "dtype": "float16",
1229
+ "format": "f32-to-bf16",
1230
+ "nbytes": 4608,
1231
+ "byteOffset": 31890432
1232
+ },
1233
+ {
1234
+ "name": "gpt_neox.layers.9.attention.dense.weight",
1235
+ "shape": [
1236
+ 768,
1237
+ 768
1238
+ ],
1239
+ "dtype": "float16",
1240
+ "format": "f32-to-bf16",
1241
+ "nbytes": 1179648,
1242
+ "byteOffset": 31895040
1243
+ },
1244
+ {
1245
+ "name": "gpt_neox.layers.9.attention.dense.bias",
1246
+ "shape": [
1247
+ 768
1248
+ ],
1249
+ "dtype": "float16",
1250
+ "format": "f32-to-bf16",
1251
+ "nbytes": 1536,
1252
+ "byteOffset": 33074688
1253
+ }
1254
+ ],
1255
+ "md5sum": "9be3a171e75708baaaf6249630cc7a0d"
1256
+ },
1257
+ {
1258
+ "dataPath": "params_shard_5.bin",
1259
+ "format": "raw-shard",
1260
+ "nbytes": 33076224,
1261
+ "records": [
1262
+ {
1263
+ "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.weight",
1264
+ "shape": [
1265
+ 3072,
1266
+ 768
1267
+ ],
1268
+ "dtype": "float16",
1269
+ "format": "f32-to-bf16",
1270
+ "nbytes": 4718592,
1271
+ "byteOffset": 0
1272
+ },
1273
+ {
1274
+ "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
1275
+ "shape": [
1276
+ 3072
1277
+ ],
1278
+ "dtype": "bfloat16",
1279
+ "format": "raw",
1280
+ "nbytes": 6144,
1281
+ "byteOffset": 4718592
1282
+ },
1283
+ {
1284
+ "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.weight",
1285
+ "shape": [
1286
+ 768,
1287
+ 3072
1288
+ ],
1289
+ "dtype": "float16",
1290
+ "format": "f32-to-bf16",
1291
+ "nbytes": 4718592,
1292
+ "byteOffset": 4724736
1293
+ },
1294
+ {
1295
+ "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
1296
+ "shape": [
1297
+ 768
1298
+ ],
1299
+ "dtype": "bfloat16",
1300
+ "format": "raw",
1301
+ "nbytes": 1536,
1302
+ "byteOffset": 9443328
1303
+ },
1304
+ {
1305
+ "name": "gpt_neox.layers.10.input_layernorm.weight",
1306
+ "shape": [
1307
+ 768
1308
+ ],
1309
+ "dtype": "float16",
1310
+ "format": "f32-to-bf16",
1311
+ "nbytes": 1536,
1312
+ "byteOffset": 9444864
1313
+ },
1314
+ {
1315
+ "name": "gpt_neox.layers.10.input_layernorm.bias",
1316
+ "shape": [
1317
+ 768
1318
+ ],
1319
+ "dtype": "float16",
1320
+ "format": "f32-to-bf16",
1321
+ "nbytes": 1536,
1322
+ "byteOffset": 9446400
1323
+ },
1324
+ {
1325
+ "name": "gpt_neox.layers.10.post_attention_layernorm.weight",
1326
+ "shape": [
1327
+ 768
1328
+ ],
1329
+ "dtype": "float16",
1330
+ "format": "f32-to-bf16",
1331
+ "nbytes": 1536,
1332
+ "byteOffset": 9447936
1333
+ },
1334
+ {
1335
+ "name": "gpt_neox.layers.10.post_attention_layernorm.bias",
1336
+ "shape": [
1337
+ 768
1338
+ ],
1339
+ "dtype": "float16",
1340
+ "format": "f32-to-bf16",
1341
+ "nbytes": 1536,
1342
+ "byteOffset": 9449472
1343
+ },
1344
+ {
1345
+ "name": "gpt_neox.layers.10.attention.query_key_value.weight",
1346
+ "shape": [
1347
+ 2304,
1348
+ 768
1349
+ ],
1350
+ "dtype": "float16",
1351
+ "format": "f32-to-bf16",
1352
+ "nbytes": 3538944,
1353
+ "byteOffset": 9451008
1354
+ },
1355
+ {
1356
+ "name": "gpt_neox.layers.10.attention.query_key_value.bias",
1357
+ "shape": [
1358
+ 2304
1359
+ ],
1360
+ "dtype": "float16",
1361
+ "format": "f32-to-bf16",
1362
+ "nbytes": 4608,
1363
+ "byteOffset": 12989952
1364
+ },
1365
+ {
1366
+ "name": "gpt_neox.layers.10.attention.dense.weight",
1367
+ "shape": [
1368
+ 768,
1369
+ 768
1370
+ ],
1371
+ "dtype": "float16",
1372
+ "format": "f32-to-bf16",
1373
+ "nbytes": 1179648,
1374
+ "byteOffset": 12994560
1375
+ },
1376
+ {
1377
+ "name": "gpt_neox.layers.10.attention.dense.bias",
1378
+ "shape": [
1379
+ 768
1380
+ ],
1381
+ "dtype": "float16",
1382
+ "format": "f32-to-bf16",
1383
+ "nbytes": 1536,
1384
+ "byteOffset": 14174208
1385
+ },
1386
+ {
1387
+ "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.weight",
1388
+ "shape": [
1389
+ 3072,
1390
+ 768
1391
+ ],
1392
+ "dtype": "float16",
1393
+ "format": "f32-to-bf16",
1394
+ "nbytes": 4718592,
1395
+ "byteOffset": 14175744
1396
+ },
1397
+ {
1398
+ "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
1399
+ "shape": [
1400
+ 3072
1401
+ ],
1402
+ "dtype": "bfloat16",
1403
+ "format": "raw",
1404
+ "nbytes": 6144,
1405
+ "byteOffset": 18894336
1406
+ },
1407
+ {
1408
+ "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.weight",
1409
+ "shape": [
1410
+ 768,
1411
+ 3072
1412
+ ],
1413
+ "dtype": "float16",
1414
+ "format": "f32-to-bf16",
1415
+ "nbytes": 4718592,
1416
+ "byteOffset": 18900480
1417
+ },
1418
+ {
1419
+ "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
1420
+ "shape": [
1421
+ 768
1422
+ ],
1423
+ "dtype": "bfloat16",
1424
+ "format": "raw",
1425
+ "nbytes": 1536,
1426
+ "byteOffset": 23619072
1427
+ },
1428
+ {
1429
+ "name": "gpt_neox.layers.11.input_layernorm.weight",
1430
+ "shape": [
1431
+ 768
1432
+ ],
1433
+ "dtype": "float16",
1434
+ "format": "f32-to-bf16",
1435
+ "nbytes": 1536,
1436
+ "byteOffset": 23620608
1437
+ },
1438
+ {
1439
+ "name": "gpt_neox.layers.11.input_layernorm.bias",
1440
+ "shape": [
1441
+ 768
1442
+ ],
1443
+ "dtype": "float16",
1444
+ "format": "f32-to-bf16",
1445
+ "nbytes": 1536,
1446
+ "byteOffset": 23622144
1447
+ },
1448
+ {
1449
+ "name": "gpt_neox.layers.11.post_attention_layernorm.weight",
1450
+ "shape": [
1451
+ 768
1452
+ ],
1453
+ "dtype": "float16",
1454
+ "format": "f32-to-bf16",
1455
+ "nbytes": 1536,
1456
+ "byteOffset": 23623680
1457
+ },
1458
+ {
1459
+ "name": "gpt_neox.layers.11.post_attention_layernorm.bias",
1460
+ "shape": [
1461
+ 768
1462
+ ],
1463
+ "dtype": "float16",
1464
+ "format": "f32-to-bf16",
1465
+ "nbytes": 1536,
1466
+ "byteOffset": 23625216
1467
+ },
1468
+ {
1469
+ "name": "gpt_neox.layers.11.attention.query_key_value.weight",
1470
+ "shape": [
1471
+ 2304,
1472
+ 768
1473
+ ],
1474
+ "dtype": "float16",
1475
+ "format": "f32-to-bf16",
1476
+ "nbytes": 3538944,
1477
+ "byteOffset": 23626752
1478
+ },
1479
+ {
1480
+ "name": "gpt_neox.layers.11.attention.query_key_value.bias",
1481
+ "shape": [
1482
+ 2304
1483
+ ],
1484
+ "dtype": "float16",
1485
+ "format": "f32-to-bf16",
1486
+ "nbytes": 4608,
1487
+ "byteOffset": 27165696
1488
+ },
1489
+ {
1490
+ "name": "gpt_neox.layers.11.attention.dense.weight",
1491
+ "shape": [
1492
+ 768,
1493
+ 768
1494
+ ],
1495
+ "dtype": "float16",
1496
+ "format": "f32-to-bf16",
1497
+ "nbytes": 1179648,
1498
+ "byteOffset": 27170304
1499
+ },
1500
+ {
1501
+ "name": "gpt_neox.layers.11.attention.dense.bias",
1502
+ "shape": [
1503
+ 768
1504
+ ],
1505
+ "dtype": "float16",
1506
+ "format": "f32-to-bf16",
1507
+ "nbytes": 1536,
1508
+ "byteOffset": 28349952
1509
+ },
1510
+ {
1511
+ "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.weight",
1512
+ "shape": [
1513
+ 3072,
1514
+ 768
1515
+ ],
1516
+ "dtype": "float16",
1517
+ "format": "f32-to-bf16",
1518
+ "nbytes": 4718592,
1519
+ "byteOffset": 28351488
1520
+ },
1521
+ {
1522
+ "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
1523
+ "shape": [
1524
+ 3072
1525
+ ],
1526
+ "dtype": "bfloat16",
1527
+ "format": "raw",
1528
+ "nbytes": 6144,
1529
+ "byteOffset": 33070080
1530
+ }
1531
+ ],
1532
+ "md5sum": "d3807c10c1e4cf1ed703d786dad97dd5"
1533
+ },
1534
+ {
1535
+ "dataPath": "params_shard_6.bin",
1536
+ "format": "raw-shard",
1537
+ "nbytes": 77266944,
1538
+ "records": [
1539
+ {
1540
+ "name": "embed_out.weight",
1541
+ "shape": [
1542
+ 50304,
1543
+ 768
1544
+ ],
1545
+ "dtype": "float16",
1546
+ "format": "f32-to-bf16",
1547
+ "nbytes": 77266944,
1548
+ "byteOffset": 0
1549
+ }
1550
+ ],
1551
+ "md5sum": "084dbad7777340578a2501ea88f226dd"
1552
+ },
1553
+ {
1554
+ "dataPath": "params_shard_7.bin",
1555
+ "format": "raw-shard",
1556
+ "nbytes": 4723200,
1557
+ "records": [
1558
+ {
1559
+ "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.weight",
1560
+ "shape": [
1561
+ 768,
1562
+ 3072
1563
+ ],
1564
+ "dtype": "float16",
1565
+ "format": "f32-to-bf16",
1566
+ "nbytes": 4718592,
1567
+ "byteOffset": 0
1568
+ },
1569
+ {
1570
+ "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
1571
+ "shape": [
1572
+ 768
1573
+ ],
1574
+ "dtype": "bfloat16",
1575
+ "format": "raw",
1576
+ "nbytes": 1536,
1577
+ "byteOffset": 4718592
1578
+ },
1579
+ {
1580
+ "name": "gpt_neox.final_layer_norm.weight",
1581
+ "shape": [
1582
+ 768
1583
+ ],
1584
+ "dtype": "float16",
1585
+ "format": "f32-to-bf16",
1586
+ "nbytes": 1536,
1587
+ "byteOffset": 4720128
1588
+ },
1589
+ {
1590
+ "name": "gpt_neox.final_layer_norm.bias",
1591
+ "shape": [
1592
+ 768
1593
+ ],
1594
+ "dtype": "float16",
1595
+ "format": "f32-to-bf16",
1596
+ "nbytes": 1536,
1597
+ "byteOffset": 4721664
1598
+ }
1599
+ ],
1600
+ "md5sum": "5290d766289dc0d43e7407c8dc728b2e"
1601
+ }
1602
+ ]
1603
+ }
ndarray-cache.json ADDED
@@ -0,0 +1,1603 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "ParamSize": 148,
4
+ "ParamBytes": 324738048.0,
5
+ "BitsPerParam": 16.00454205660538
6
+ },
7
+ "records": [
8
+ {
9
+ "dataPath": "params_shard_0.bin",
10
+ "format": "raw-shard",
11
+ "nbytes": 77266944,
12
+ "records": [
13
+ {
14
+ "name": "gpt_neox.embed_in.weight",
15
+ "shape": [
16
+ 50304,
17
+ 768
18
+ ],
19
+ "dtype": "float16",
20
+ "format": "f32-to-bf16",
21
+ "nbytes": 77266944,
22
+ "byteOffset": 0
23
+ }
24
+ ],
25
+ "md5sum": "2481121002c12eed2951b729fa17b644"
26
+ },
27
+ {
28
+ "dataPath": "params_shard_1.bin",
29
+ "format": "raw-shard",
30
+ "nbytes": 33082368,
31
+ "records": [
32
+ {
33
+ "name": "gpt_neox.layers.0.input_layernorm.weight",
34
+ "shape": [
35
+ 768
36
+ ],
37
+ "dtype": "float16",
38
+ "format": "f32-to-bf16",
39
+ "nbytes": 1536,
40
+ "byteOffset": 0
41
+ },
42
+ {
43
+ "name": "gpt_neox.layers.0.input_layernorm.bias",
44
+ "shape": [
45
+ 768
46
+ ],
47
+ "dtype": "float16",
48
+ "format": "f32-to-bf16",
49
+ "nbytes": 1536,
50
+ "byteOffset": 1536
51
+ },
52
+ {
53
+ "name": "gpt_neox.layers.0.post_attention_layernorm.weight",
54
+ "shape": [
55
+ 768
56
+ ],
57
+ "dtype": "float16",
58
+ "format": "f32-to-bf16",
59
+ "nbytes": 1536,
60
+ "byteOffset": 3072
61
+ },
62
+ {
63
+ "name": "gpt_neox.layers.0.post_attention_layernorm.bias",
64
+ "shape": [
65
+ 768
66
+ ],
67
+ "dtype": "float16",
68
+ "format": "f32-to-bf16",
69
+ "nbytes": 1536,
70
+ "byteOffset": 4608
71
+ },
72
+ {
73
+ "name": "gpt_neox.layers.0.attention.query_key_value.weight",
74
+ "shape": [
75
+ 2304,
76
+ 768
77
+ ],
78
+ "dtype": "float16",
79
+ "format": "f32-to-bf16",
80
+ "nbytes": 3538944,
81
+ "byteOffset": 6144
82
+ },
83
+ {
84
+ "name": "gpt_neox.layers.0.attention.query_key_value.bias",
85
+ "shape": [
86
+ 2304
87
+ ],
88
+ "dtype": "float16",
89
+ "format": "f32-to-bf16",
90
+ "nbytes": 4608,
91
+ "byteOffset": 3545088
92
+ },
93
+ {
94
+ "name": "gpt_neox.layers.0.attention.dense.weight",
95
+ "shape": [
96
+ 768,
97
+ 768
98
+ ],
99
+ "dtype": "float16",
100
+ "format": "f32-to-bf16",
101
+ "nbytes": 1179648,
102
+ "byteOffset": 3549696
103
+ },
104
+ {
105
+ "name": "gpt_neox.layers.0.attention.dense.bias",
106
+ "shape": [
107
+ 768
108
+ ],
109
+ "dtype": "float16",
110
+ "format": "f32-to-bf16",
111
+ "nbytes": 1536,
112
+ "byteOffset": 4729344
113
+ },
114
+ {
115
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight",
116
+ "shape": [
117
+ 3072,
118
+ 768
119
+ ],
120
+ "dtype": "float16",
121
+ "format": "f32-to-bf16",
122
+ "nbytes": 4718592,
123
+ "byteOffset": 4730880
124
+ },
125
+ {
126
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
127
+ "shape": [
128
+ 3072
129
+ ],
130
+ "dtype": "float32",
131
+ "format": "f32-to-bf16",
132
+ "nbytes": 6144,
133
+ "byteOffset": 9449472
134
+ },
135
+ {
136
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight",
137
+ "shape": [
138
+ 768,
139
+ 3072
140
+ ],
141
+ "dtype": "float16",
142
+ "format": "f32-to-bf16",
143
+ "nbytes": 4718592,
144
+ "byteOffset": 9455616
145
+ },
146
+ {
147
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
148
+ "shape": [
149
+ 768
150
+ ],
151
+ "dtype": "float32",
152
+ "format": "f32-to-bf16",
153
+ "nbytes": 1536,
154
+ "byteOffset": 14174208
155
+ },
156
+ {
157
+ "name": "gpt_neox.layers.1.input_layernorm.weight",
158
+ "shape": [
159
+ 768
160
+ ],
161
+ "dtype": "float16",
162
+ "format": "f32-to-bf16",
163
+ "nbytes": 1536,
164
+ "byteOffset": 14175744
165
+ },
166
+ {
167
+ "name": "gpt_neox.layers.1.input_layernorm.bias",
168
+ "shape": [
169
+ 768
170
+ ],
171
+ "dtype": "float16",
172
+ "format": "f32-to-bf16",
173
+ "nbytes": 1536,
174
+ "byteOffset": 14177280
175
+ },
176
+ {
177
+ "name": "gpt_neox.layers.1.post_attention_layernorm.weight",
178
+ "shape": [
179
+ 768
180
+ ],
181
+ "dtype": "float16",
182
+ "format": "f32-to-bf16",
183
+ "nbytes": 1536,
184
+ "byteOffset": 14178816
185
+ },
186
+ {
187
+ "name": "gpt_neox.layers.1.post_attention_layernorm.bias",
188
+ "shape": [
189
+ 768
190
+ ],
191
+ "dtype": "float16",
192
+ "format": "f32-to-bf16",
193
+ "nbytes": 1536,
194
+ "byteOffset": 14180352
195
+ },
196
+ {
197
+ "name": "gpt_neox.layers.1.attention.query_key_value.weight",
198
+ "shape": [
199
+ 2304,
200
+ 768
201
+ ],
202
+ "dtype": "float16",
203
+ "format": "f32-to-bf16",
204
+ "nbytes": 3538944,
205
+ "byteOffset": 14181888
206
+ },
207
+ {
208
+ "name": "gpt_neox.layers.1.attention.query_key_value.bias",
209
+ "shape": [
210
+ 2304
211
+ ],
212
+ "dtype": "float16",
213
+ "format": "f32-to-bf16",
214
+ "nbytes": 4608,
215
+ "byteOffset": 17720832
216
+ },
217
+ {
218
+ "name": "gpt_neox.layers.1.attention.dense.weight",
219
+ "shape": [
220
+ 768,
221
+ 768
222
+ ],
223
+ "dtype": "float16",
224
+ "format": "f32-to-bf16",
225
+ "nbytes": 1179648,
226
+ "byteOffset": 17725440
227
+ },
228
+ {
229
+ "name": "gpt_neox.layers.1.attention.dense.bias",
230
+ "shape": [
231
+ 768
232
+ ],
233
+ "dtype": "float16",
234
+ "format": "f32-to-bf16",
235
+ "nbytes": 1536,
236
+ "byteOffset": 18905088
237
+ },
238
+ {
239
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight",
240
+ "shape": [
241
+ 3072,
242
+ 768
243
+ ],
244
+ "dtype": "float16",
245
+ "format": "f32-to-bf16",
246
+ "nbytes": 4718592,
247
+ "byteOffset": 18906624
248
+ },
249
+ {
250
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
251
+ "shape": [
252
+ 3072
253
+ ],
254
+ "dtype": "float32",
255
+ "format": "f32-to-bf16",
256
+ "nbytes": 6144,
257
+ "byteOffset": 23625216
258
+ },
259
+ {
260
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight",
261
+ "shape": [
262
+ 768,
263
+ 3072
264
+ ],
265
+ "dtype": "float16",
266
+ "format": "f32-to-bf16",
267
+ "nbytes": 4718592,
268
+ "byteOffset": 23631360
269
+ },
270
+ {
271
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
272
+ "shape": [
273
+ 768
274
+ ],
275
+ "dtype": "float32",
276
+ "format": "f32-to-bf16",
277
+ "nbytes": 1536,
278
+ "byteOffset": 28349952
279
+ },
280
+ {
281
+ "name": "gpt_neox.layers.2.input_layernorm.weight",
282
+ "shape": [
283
+ 768
284
+ ],
285
+ "dtype": "float16",
286
+ "format": "f32-to-bf16",
287
+ "nbytes": 1536,
288
+ "byteOffset": 28351488
289
+ },
290
+ {
291
+ "name": "gpt_neox.layers.2.input_layernorm.bias",
292
+ "shape": [
293
+ 768
294
+ ],
295
+ "dtype": "float16",
296
+ "format": "f32-to-bf16",
297
+ "nbytes": 1536,
298
+ "byteOffset": 28353024
299
+ },
300
+ {
301
+ "name": "gpt_neox.layers.2.post_attention_layernorm.weight",
302
+ "shape": [
303
+ 768
304
+ ],
305
+ "dtype": "float16",
306
+ "format": "f32-to-bf16",
307
+ "nbytes": 1536,
308
+ "byteOffset": 28354560
309
+ },
310
+ {
311
+ "name": "gpt_neox.layers.2.post_attention_layernorm.bias",
312
+ "shape": [
313
+ 768
314
+ ],
315
+ "dtype": "float16",
316
+ "format": "f32-to-bf16",
317
+ "nbytes": 1536,
318
+ "byteOffset": 28356096
319
+ },
320
+ {
321
+ "name": "gpt_neox.layers.2.attention.query_key_value.weight",
322
+ "shape": [
323
+ 2304,
324
+ 768
325
+ ],
326
+ "dtype": "float16",
327
+ "format": "f32-to-bf16",
328
+ "nbytes": 3538944,
329
+ "byteOffset": 28357632
330
+ },
331
+ {
332
+ "name": "gpt_neox.layers.2.attention.query_key_value.bias",
333
+ "shape": [
334
+ 2304
335
+ ],
336
+ "dtype": "float16",
337
+ "format": "f32-to-bf16",
338
+ "nbytes": 4608,
339
+ "byteOffset": 31896576
340
+ },
341
+ {
342
+ "name": "gpt_neox.layers.2.attention.dense.weight",
343
+ "shape": [
344
+ 768,
345
+ 768
346
+ ],
347
+ "dtype": "float16",
348
+ "format": "f32-to-bf16",
349
+ "nbytes": 1179648,
350
+ "byteOffset": 31901184
351
+ },
352
+ {
353
+ "name": "gpt_neox.layers.2.attention.dense.bias",
354
+ "shape": [
355
+ 768
356
+ ],
357
+ "dtype": "float16",
358
+ "format": "f32-to-bf16",
359
+ "nbytes": 1536,
360
+ "byteOffset": 33080832
361
+ }
362
+ ],
363
+ "md5sum": "f152a892aaec5607fd2b89419435780a"
364
+ },
365
+ {
366
+ "dataPath": "params_shard_2.bin",
367
+ "format": "raw-shard",
368
+ "nbytes": 33076224,
369
+ "records": [
370
+ {
371
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight",
372
+ "shape": [
373
+ 3072,
374
+ 768
375
+ ],
376
+ "dtype": "float16",
377
+ "format": "f32-to-bf16",
378
+ "nbytes": 4718592,
379
+ "byteOffset": 0
380
+ },
381
+ {
382
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
383
+ "shape": [
384
+ 3072
385
+ ],
386
+ "dtype": "float32",
387
+ "format": "f32-to-bf16",
388
+ "nbytes": 6144,
389
+ "byteOffset": 4718592
390
+ },
391
+ {
392
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight",
393
+ "shape": [
394
+ 768,
395
+ 3072
396
+ ],
397
+ "dtype": "float16",
398
+ "format": "f32-to-bf16",
399
+ "nbytes": 4718592,
400
+ "byteOffset": 4724736
401
+ },
402
+ {
403
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
404
+ "shape": [
405
+ 768
406
+ ],
407
+ "dtype": "float32",
408
+ "format": "f32-to-bf16",
409
+ "nbytes": 1536,
410
+ "byteOffset": 9443328
411
+ },
412
+ {
413
+ "name": "gpt_neox.layers.3.input_layernorm.weight",
414
+ "shape": [
415
+ 768
416
+ ],
417
+ "dtype": "float16",
418
+ "format": "f32-to-bf16",
419
+ "nbytes": 1536,
420
+ "byteOffset": 9444864
421
+ },
422
+ {
423
+ "name": "gpt_neox.layers.3.input_layernorm.bias",
424
+ "shape": [
425
+ 768
426
+ ],
427
+ "dtype": "float16",
428
+ "format": "f32-to-bf16",
429
+ "nbytes": 1536,
430
+ "byteOffset": 9446400
431
+ },
432
+ {
433
+ "name": "gpt_neox.layers.3.post_attention_layernorm.weight",
434
+ "shape": [
435
+ 768
436
+ ],
437
+ "dtype": "float16",
438
+ "format": "f32-to-bf16",
439
+ "nbytes": 1536,
440
+ "byteOffset": 9447936
441
+ },
442
+ {
443
+ "name": "gpt_neox.layers.3.post_attention_layernorm.bias",
444
+ "shape": [
445
+ 768
446
+ ],
447
+ "dtype": "float16",
448
+ "format": "f32-to-bf16",
449
+ "nbytes": 1536,
450
+ "byteOffset": 9449472
451
+ },
452
+ {
453
+ "name": "gpt_neox.layers.3.attention.query_key_value.weight",
454
+ "shape": [
455
+ 2304,
456
+ 768
457
+ ],
458
+ "dtype": "float16",
459
+ "format": "f32-to-bf16",
460
+ "nbytes": 3538944,
461
+ "byteOffset": 9451008
462
+ },
463
+ {
464
+ "name": "gpt_neox.layers.3.attention.query_key_value.bias",
465
+ "shape": [
466
+ 2304
467
+ ],
468
+ "dtype": "float16",
469
+ "format": "f32-to-bf16",
470
+ "nbytes": 4608,
471
+ "byteOffset": 12989952
472
+ },
473
+ {
474
+ "name": "gpt_neox.layers.3.attention.dense.weight",
475
+ "shape": [
476
+ 768,
477
+ 768
478
+ ],
479
+ "dtype": "float16",
480
+ "format": "f32-to-bf16",
481
+ "nbytes": 1179648,
482
+ "byteOffset": 12994560
483
+ },
484
+ {
485
+ "name": "gpt_neox.layers.3.attention.dense.bias",
486
+ "shape": [
487
+ 768
488
+ ],
489
+ "dtype": "float16",
490
+ "format": "f32-to-bf16",
491
+ "nbytes": 1536,
492
+ "byteOffset": 14174208
493
+ },
494
+ {
495
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight",
496
+ "shape": [
497
+ 3072,
498
+ 768
499
+ ],
500
+ "dtype": "float16",
501
+ "format": "f32-to-bf16",
502
+ "nbytes": 4718592,
503
+ "byteOffset": 14175744
504
+ },
505
+ {
506
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
507
+ "shape": [
508
+ 3072
509
+ ],
510
+ "dtype": "float32",
511
+ "format": "f32-to-bf16",
512
+ "nbytes": 6144,
513
+ "byteOffset": 18894336
514
+ },
515
+ {
516
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight",
517
+ "shape": [
518
+ 768,
519
+ 3072
520
+ ],
521
+ "dtype": "float16",
522
+ "format": "f32-to-bf16",
523
+ "nbytes": 4718592,
524
+ "byteOffset": 18900480
525
+ },
526
+ {
527
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
528
+ "shape": [
529
+ 768
530
+ ],
531
+ "dtype": "float32",
532
+ "format": "f32-to-bf16",
533
+ "nbytes": 1536,
534
+ "byteOffset": 23619072
535
+ },
536
+ {
537
+ "name": "gpt_neox.layers.4.input_layernorm.weight",
538
+ "shape": [
539
+ 768
540
+ ],
541
+ "dtype": "float16",
542
+ "format": "f32-to-bf16",
543
+ "nbytes": 1536,
544
+ "byteOffset": 23620608
545
+ },
546
+ {
547
+ "name": "gpt_neox.layers.4.input_layernorm.bias",
548
+ "shape": [
549
+ 768
550
+ ],
551
+ "dtype": "float16",
552
+ "format": "f32-to-bf16",
553
+ "nbytes": 1536,
554
+ "byteOffset": 23622144
555
+ },
556
+ {
557
+ "name": "gpt_neox.layers.4.post_attention_layernorm.weight",
558
+ "shape": [
559
+ 768
560
+ ],
561
+ "dtype": "float16",
562
+ "format": "f32-to-bf16",
563
+ "nbytes": 1536,
564
+ "byteOffset": 23623680
565
+ },
566
+ {
567
+ "name": "gpt_neox.layers.4.post_attention_layernorm.bias",
568
+ "shape": [
569
+ 768
570
+ ],
571
+ "dtype": "float16",
572
+ "format": "f32-to-bf16",
573
+ "nbytes": 1536,
574
+ "byteOffset": 23625216
575
+ },
576
+ {
577
+ "name": "gpt_neox.layers.4.attention.query_key_value.weight",
578
+ "shape": [
579
+ 2304,
580
+ 768
581
+ ],
582
+ "dtype": "float16",
583
+ "format": "f32-to-bf16",
584
+ "nbytes": 3538944,
585
+ "byteOffset": 23626752
586
+ },
587
+ {
588
+ "name": "gpt_neox.layers.4.attention.query_key_value.bias",
589
+ "shape": [
590
+ 2304
591
+ ],
592
+ "dtype": "float16",
593
+ "format": "f32-to-bf16",
594
+ "nbytes": 4608,
595
+ "byteOffset": 27165696
596
+ },
597
+ {
598
+ "name": "gpt_neox.layers.4.attention.dense.weight",
599
+ "shape": [
600
+ 768,
601
+ 768
602
+ ],
603
+ "dtype": "float16",
604
+ "format": "f32-to-bf16",
605
+ "nbytes": 1179648,
606
+ "byteOffset": 27170304
607
+ },
608
+ {
609
+ "name": "gpt_neox.layers.4.attention.dense.bias",
610
+ "shape": [
611
+ 768
612
+ ],
613
+ "dtype": "float16",
614
+ "format": "f32-to-bf16",
615
+ "nbytes": 1536,
616
+ "byteOffset": 28349952
617
+ },
618
+ {
619
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight",
620
+ "shape": [
621
+ 3072,
622
+ 768
623
+ ],
624
+ "dtype": "float16",
625
+ "format": "f32-to-bf16",
626
+ "nbytes": 4718592,
627
+ "byteOffset": 28351488
628
+ },
629
+ {
630
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
631
+ "shape": [
632
+ 3072
633
+ ],
634
+ "dtype": "float32",
635
+ "format": "f32-to-bf16",
636
+ "nbytes": 6144,
637
+ "byteOffset": 33070080
638
+ }
639
+ ],
640
+ "md5sum": "57ece0af7da914417b0f9080c0bdac26"
641
+ },
642
+ {
643
+ "dataPath": "params_shard_3.bin",
644
+ "format": "raw-shard",
645
+ "nbytes": 33077760,
646
+ "records": [
647
+ {
648
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight",
649
+ "shape": [
650
+ 768,
651
+ 3072
652
+ ],
653
+ "dtype": "float16",
654
+ "format": "f32-to-bf16",
655
+ "nbytes": 4718592,
656
+ "byteOffset": 0
657
+ },
658
+ {
659
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
660
+ "shape": [
661
+ 768
662
+ ],
663
+ "dtype": "float32",
664
+ "format": "f32-to-bf16",
665
+ "nbytes": 1536,
666
+ "byteOffset": 4718592
667
+ },
668
+ {
669
+ "name": "gpt_neox.layers.5.input_layernorm.weight",
670
+ "shape": [
671
+ 768
672
+ ],
673
+ "dtype": "float16",
674
+ "format": "f32-to-bf16",
675
+ "nbytes": 1536,
676
+ "byteOffset": 4720128
677
+ },
678
+ {
679
+ "name": "gpt_neox.layers.5.input_layernorm.bias",
680
+ "shape": [
681
+ 768
682
+ ],
683
+ "dtype": "float16",
684
+ "format": "f32-to-bf16",
685
+ "nbytes": 1536,
686
+ "byteOffset": 4721664
687
+ },
688
+ {
689
+ "name": "gpt_neox.layers.5.post_attention_layernorm.weight",
690
+ "shape": [
691
+ 768
692
+ ],
693
+ "dtype": "float16",
694
+ "format": "f32-to-bf16",
695
+ "nbytes": 1536,
696
+ "byteOffset": 4723200
697
+ },
698
+ {
699
+ "name": "gpt_neox.layers.5.post_attention_layernorm.bias",
700
+ "shape": [
701
+ 768
702
+ ],
703
+ "dtype": "float16",
704
+ "format": "f32-to-bf16",
705
+ "nbytes": 1536,
706
+ "byteOffset": 4724736
707
+ },
708
+ {
709
+ "name": "gpt_neox.layers.5.attention.query_key_value.weight",
710
+ "shape": [
711
+ 2304,
712
+ 768
713
+ ],
714
+ "dtype": "float16",
715
+ "format": "f32-to-bf16",
716
+ "nbytes": 3538944,
717
+ "byteOffset": 4726272
718
+ },
719
+ {
720
+ "name": "gpt_neox.layers.5.attention.query_key_value.bias",
721
+ "shape": [
722
+ 2304
723
+ ],
724
+ "dtype": "float16",
725
+ "format": "f32-to-bf16",
726
+ "nbytes": 4608,
727
+ "byteOffset": 8265216
728
+ },
729
+ {
730
+ "name": "gpt_neox.layers.5.attention.dense.weight",
731
+ "shape": [
732
+ 768,
733
+ 768
734
+ ],
735
+ "dtype": "float16",
736
+ "format": "f32-to-bf16",
737
+ "nbytes": 1179648,
738
+ "byteOffset": 8269824
739
+ },
740
+ {
741
+ "name": "gpt_neox.layers.5.attention.dense.bias",
742
+ "shape": [
743
+ 768
744
+ ],
745
+ "dtype": "float16",
746
+ "format": "f32-to-bf16",
747
+ "nbytes": 1536,
748
+ "byteOffset": 9449472
749
+ },
750
+ {
751
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight",
752
+ "shape": [
753
+ 3072,
754
+ 768
755
+ ],
756
+ "dtype": "float16",
757
+ "format": "f32-to-bf16",
758
+ "nbytes": 4718592,
759
+ "byteOffset": 9451008
760
+ },
761
+ {
762
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
763
+ "shape": [
764
+ 3072
765
+ ],
766
+ "dtype": "float32",
767
+ "format": "f32-to-bf16",
768
+ "nbytes": 6144,
769
+ "byteOffset": 14169600
770
+ },
771
+ {
772
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight",
773
+ "shape": [
774
+ 768,
775
+ 3072
776
+ ],
777
+ "dtype": "float16",
778
+ "format": "f32-to-bf16",
779
+ "nbytes": 4718592,
780
+ "byteOffset": 14175744
781
+ },
782
+ {
783
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
784
+ "shape": [
785
+ 768
786
+ ],
787
+ "dtype": "float32",
788
+ "format": "f32-to-bf16",
789
+ "nbytes": 1536,
790
+ "byteOffset": 18894336
791
+ },
792
+ {
793
+ "name": "gpt_neox.layers.6.input_layernorm.weight",
794
+ "shape": [
795
+ 768
796
+ ],
797
+ "dtype": "float16",
798
+ "format": "f32-to-bf16",
799
+ "nbytes": 1536,
800
+ "byteOffset": 18895872
801
+ },
802
+ {
803
+ "name": "gpt_neox.layers.6.input_layernorm.bias",
804
+ "shape": [
805
+ 768
806
+ ],
807
+ "dtype": "float16",
808
+ "format": "f32-to-bf16",
809
+ "nbytes": 1536,
810
+ "byteOffset": 18897408
811
+ },
812
+ {
813
+ "name": "gpt_neox.layers.6.post_attention_layernorm.weight",
814
+ "shape": [
815
+ 768
816
+ ],
817
+ "dtype": "float16",
818
+ "format": "f32-to-bf16",
819
+ "nbytes": 1536,
820
+ "byteOffset": 18898944
821
+ },
822
+ {
823
+ "name": "gpt_neox.layers.6.post_attention_layernorm.bias",
824
+ "shape": [
825
+ 768
826
+ ],
827
+ "dtype": "float16",
828
+ "format": "f32-to-bf16",
829
+ "nbytes": 1536,
830
+ "byteOffset": 18900480
831
+ },
832
+ {
833
+ "name": "gpt_neox.layers.6.attention.query_key_value.weight",
834
+ "shape": [
835
+ 2304,
836
+ 768
837
+ ],
838
+ "dtype": "float16",
839
+ "format": "f32-to-bf16",
840
+ "nbytes": 3538944,
841
+ "byteOffset": 18902016
842
+ },
843
+ {
844
+ "name": "gpt_neox.layers.6.attention.query_key_value.bias",
845
+ "shape": [
846
+ 2304
847
+ ],
848
+ "dtype": "float16",
849
+ "format": "f32-to-bf16",
850
+ "nbytes": 4608,
851
+ "byteOffset": 22440960
852
+ },
853
+ {
854
+ "name": "gpt_neox.layers.6.attention.dense.weight",
855
+ "shape": [
856
+ 768,
857
+ 768
858
+ ],
859
+ "dtype": "float16",
860
+ "format": "f32-to-bf16",
861
+ "nbytes": 1179648,
862
+ "byteOffset": 22445568
863
+ },
864
+ {
865
+ "name": "gpt_neox.layers.6.attention.dense.bias",
866
+ "shape": [
867
+ 768
868
+ ],
869
+ "dtype": "float16",
870
+ "format": "f32-to-bf16",
871
+ "nbytes": 1536,
872
+ "byteOffset": 23625216
873
+ },
874
+ {
875
+ "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.weight",
876
+ "shape": [
877
+ 3072,
878
+ 768
879
+ ],
880
+ "dtype": "float16",
881
+ "format": "f32-to-bf16",
882
+ "nbytes": 4718592,
883
+ "byteOffset": 23626752
884
+ },
885
+ {
886
+ "name": "gpt_neox.layers.6.mlp.dense_h_to_4h.bias",
887
+ "shape": [
888
+ 3072
889
+ ],
890
+ "dtype": "float32",
891
+ "format": "f32-to-bf16",
892
+ "nbytes": 6144,
893
+ "byteOffset": 28345344
894
+ },
895
+ {
896
+ "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.weight",
897
+ "shape": [
898
+ 768,
899
+ 3072
900
+ ],
901
+ "dtype": "float16",
902
+ "format": "f32-to-bf16",
903
+ "nbytes": 4718592,
904
+ "byteOffset": 28351488
905
+ },
906
+ {
907
+ "name": "gpt_neox.layers.6.mlp.dense_4h_to_h.bias",
908
+ "shape": [
909
+ 768
910
+ ],
911
+ "dtype": "float32",
912
+ "format": "f32-to-bf16",
913
+ "nbytes": 1536,
914
+ "byteOffset": 33070080
915
+ },
916
+ {
917
+ "name": "gpt_neox.layers.7.input_layernorm.weight",
918
+ "shape": [
919
+ 768
920
+ ],
921
+ "dtype": "float16",
922
+ "format": "f32-to-bf16",
923
+ "nbytes": 1536,
924
+ "byteOffset": 33071616
925
+ },
926
+ {
927
+ "name": "gpt_neox.layers.7.input_layernorm.bias",
928
+ "shape": [
929
+ 768
930
+ ],
931
+ "dtype": "float16",
932
+ "format": "f32-to-bf16",
933
+ "nbytes": 1536,
934
+ "byteOffset": 33073152
935
+ },
936
+ {
937
+ "name": "gpt_neox.layers.7.post_attention_layernorm.weight",
938
+ "shape": [
939
+ 768
940
+ ],
941
+ "dtype": "float16",
942
+ "format": "f32-to-bf16",
943
+ "nbytes": 1536,
944
+ "byteOffset": 33074688
945
+ },
946
+ {
947
+ "name": "gpt_neox.layers.7.post_attention_layernorm.bias",
948
+ "shape": [
949
+ 768
950
+ ],
951
+ "dtype": "float16",
952
+ "format": "f32-to-bf16",
953
+ "nbytes": 1536,
954
+ "byteOffset": 33076224
955
+ }
956
+ ],
957
+ "md5sum": "773204e0e5615a84b4109b95d21abb1e"
958
+ },
959
+ {
960
+ "dataPath": "params_shard_4.bin",
961
+ "format": "raw-shard",
962
+ "nbytes": 33076224,
963
+ "records": [
964
+ {
965
+ "name": "gpt_neox.layers.7.attention.query_key_value.weight",
966
+ "shape": [
967
+ 2304,
968
+ 768
969
+ ],
970
+ "dtype": "float16",
971
+ "format": "f32-to-bf16",
972
+ "nbytes": 3538944,
973
+ "byteOffset": 0
974
+ },
975
+ {
976
+ "name": "gpt_neox.layers.7.attention.query_key_value.bias",
977
+ "shape": [
978
+ 2304
979
+ ],
980
+ "dtype": "float16",
981
+ "format": "f32-to-bf16",
982
+ "nbytes": 4608,
983
+ "byteOffset": 3538944
984
+ },
985
+ {
986
+ "name": "gpt_neox.layers.7.attention.dense.weight",
987
+ "shape": [
988
+ 768,
989
+ 768
990
+ ],
991
+ "dtype": "float16",
992
+ "format": "f32-to-bf16",
993
+ "nbytes": 1179648,
994
+ "byteOffset": 3543552
995
+ },
996
+ {
997
+ "name": "gpt_neox.layers.7.attention.dense.bias",
998
+ "shape": [
999
+ 768
1000
+ ],
1001
+ "dtype": "float16",
1002
+ "format": "f32-to-bf16",
1003
+ "nbytes": 1536,
1004
+ "byteOffset": 4723200
1005
+ },
1006
+ {
1007
+ "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.weight",
1008
+ "shape": [
1009
+ 3072,
1010
+ 768
1011
+ ],
1012
+ "dtype": "float16",
1013
+ "format": "f32-to-bf16",
1014
+ "nbytes": 4718592,
1015
+ "byteOffset": 4724736
1016
+ },
1017
+ {
1018
+ "name": "gpt_neox.layers.7.mlp.dense_h_to_4h.bias",
1019
+ "shape": [
1020
+ 3072
1021
+ ],
1022
+ "dtype": "float32",
1023
+ "format": "f32-to-bf16",
1024
+ "nbytes": 6144,
1025
+ "byteOffset": 9443328
1026
+ },
1027
+ {
1028
+ "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.weight",
1029
+ "shape": [
1030
+ 768,
1031
+ 3072
1032
+ ],
1033
+ "dtype": "float16",
1034
+ "format": "f32-to-bf16",
1035
+ "nbytes": 4718592,
1036
+ "byteOffset": 9449472
1037
+ },
1038
+ {
1039
+ "name": "gpt_neox.layers.7.mlp.dense_4h_to_h.bias",
1040
+ "shape": [
1041
+ 768
1042
+ ],
1043
+ "dtype": "float32",
1044
+ "format": "f32-to-bf16",
1045
+ "nbytes": 1536,
1046
+ "byteOffset": 14168064
1047
+ },
1048
+ {
1049
+ "name": "gpt_neox.layers.8.input_layernorm.weight",
1050
+ "shape": [
1051
+ 768
1052
+ ],
1053
+ "dtype": "float16",
1054
+ "format": "f32-to-bf16",
1055
+ "nbytes": 1536,
1056
+ "byteOffset": 14169600
1057
+ },
1058
+ {
1059
+ "name": "gpt_neox.layers.8.input_layernorm.bias",
1060
+ "shape": [
1061
+ 768
1062
+ ],
1063
+ "dtype": "float16",
1064
+ "format": "f32-to-bf16",
1065
+ "nbytes": 1536,
1066
+ "byteOffset": 14171136
1067
+ },
1068
+ {
1069
+ "name": "gpt_neox.layers.8.post_attention_layernorm.weight",
1070
+ "shape": [
1071
+ 768
1072
+ ],
1073
+ "dtype": "float16",
1074
+ "format": "f32-to-bf16",
1075
+ "nbytes": 1536,
1076
+ "byteOffset": 14172672
1077
+ },
1078
+ {
1079
+ "name": "gpt_neox.layers.8.post_attention_layernorm.bias",
1080
+ "shape": [
1081
+ 768
1082
+ ],
1083
+ "dtype": "float16",
1084
+ "format": "f32-to-bf16",
1085
+ "nbytes": 1536,
1086
+ "byteOffset": 14174208
1087
+ },
1088
+ {
1089
+ "name": "gpt_neox.layers.8.attention.query_key_value.weight",
1090
+ "shape": [
1091
+ 2304,
1092
+ 768
1093
+ ],
1094
+ "dtype": "float16",
1095
+ "format": "f32-to-bf16",
1096
+ "nbytes": 3538944,
1097
+ "byteOffset": 14175744
1098
+ },
1099
+ {
1100
+ "name": "gpt_neox.layers.8.attention.query_key_value.bias",
1101
+ "shape": [
1102
+ 2304
1103
+ ],
1104
+ "dtype": "float16",
1105
+ "format": "f32-to-bf16",
1106
+ "nbytes": 4608,
1107
+ "byteOffset": 17714688
1108
+ },
1109
+ {
1110
+ "name": "gpt_neox.layers.8.attention.dense.weight",
1111
+ "shape": [
1112
+ 768,
1113
+ 768
1114
+ ],
1115
+ "dtype": "float16",
1116
+ "format": "f32-to-bf16",
1117
+ "nbytes": 1179648,
1118
+ "byteOffset": 17719296
1119
+ },
1120
+ {
1121
+ "name": "gpt_neox.layers.8.attention.dense.bias",
1122
+ "shape": [
1123
+ 768
1124
+ ],
1125
+ "dtype": "float16",
1126
+ "format": "f32-to-bf16",
1127
+ "nbytes": 1536,
1128
+ "byteOffset": 18898944
1129
+ },
1130
+ {
1131
+ "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.weight",
1132
+ "shape": [
1133
+ 3072,
1134
+ 768
1135
+ ],
1136
+ "dtype": "float16",
1137
+ "format": "f32-to-bf16",
1138
+ "nbytes": 4718592,
1139
+ "byteOffset": 18900480
1140
+ },
1141
+ {
1142
+ "name": "gpt_neox.layers.8.mlp.dense_h_to_4h.bias",
1143
+ "shape": [
1144
+ 3072
1145
+ ],
1146
+ "dtype": "float32",
1147
+ "format": "f32-to-bf16",
1148
+ "nbytes": 6144,
1149
+ "byteOffset": 23619072
1150
+ },
1151
+ {
1152
+ "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.weight",
1153
+ "shape": [
1154
+ 768,
1155
+ 3072
1156
+ ],
1157
+ "dtype": "float16",
1158
+ "format": "f32-to-bf16",
1159
+ "nbytes": 4718592,
1160
+ "byteOffset": 23625216
1161
+ },
1162
+ {
1163
+ "name": "gpt_neox.layers.8.mlp.dense_4h_to_h.bias",
1164
+ "shape": [
1165
+ 768
1166
+ ],
1167
+ "dtype": "float32",
1168
+ "format": "f32-to-bf16",
1169
+ "nbytes": 1536,
1170
+ "byteOffset": 28343808
1171
+ },
1172
+ {
1173
+ "name": "gpt_neox.layers.9.input_layernorm.weight",
1174
+ "shape": [
1175
+ 768
1176
+ ],
1177
+ "dtype": "float16",
1178
+ "format": "f32-to-bf16",
1179
+ "nbytes": 1536,
1180
+ "byteOffset": 28345344
1181
+ },
1182
+ {
1183
+ "name": "gpt_neox.layers.9.input_layernorm.bias",
1184
+ "shape": [
1185
+ 768
1186
+ ],
1187
+ "dtype": "float16",
1188
+ "format": "f32-to-bf16",
1189
+ "nbytes": 1536,
1190
+ "byteOffset": 28346880
1191
+ },
1192
+ {
1193
+ "name": "gpt_neox.layers.9.post_attention_layernorm.weight",
1194
+ "shape": [
1195
+ 768
1196
+ ],
1197
+ "dtype": "float16",
1198
+ "format": "f32-to-bf16",
1199
+ "nbytes": 1536,
1200
+ "byteOffset": 28348416
1201
+ },
1202
+ {
1203
+ "name": "gpt_neox.layers.9.post_attention_layernorm.bias",
1204
+ "shape": [
1205
+ 768
1206
+ ],
1207
+ "dtype": "float16",
1208
+ "format": "f32-to-bf16",
1209
+ "nbytes": 1536,
1210
+ "byteOffset": 28349952
1211
+ },
1212
+ {
1213
+ "name": "gpt_neox.layers.9.attention.query_key_value.weight",
1214
+ "shape": [
1215
+ 2304,
1216
+ 768
1217
+ ],
1218
+ "dtype": "float16",
1219
+ "format": "f32-to-bf16",
1220
+ "nbytes": 3538944,
1221
+ "byteOffset": 28351488
1222
+ },
1223
+ {
1224
+ "name": "gpt_neox.layers.9.attention.query_key_value.bias",
1225
+ "shape": [
1226
+ 2304
1227
+ ],
1228
+ "dtype": "float16",
1229
+ "format": "f32-to-bf16",
1230
+ "nbytes": 4608,
1231
+ "byteOffset": 31890432
1232
+ },
1233
+ {
1234
+ "name": "gpt_neox.layers.9.attention.dense.weight",
1235
+ "shape": [
1236
+ 768,
1237
+ 768
1238
+ ],
1239
+ "dtype": "float16",
1240
+ "format": "f32-to-bf16",
1241
+ "nbytes": 1179648,
1242
+ "byteOffset": 31895040
1243
+ },
1244
+ {
1245
+ "name": "gpt_neox.layers.9.attention.dense.bias",
1246
+ "shape": [
1247
+ 768
1248
+ ],
1249
+ "dtype": "float16",
1250
+ "format": "f32-to-bf16",
1251
+ "nbytes": 1536,
1252
+ "byteOffset": 33074688
1253
+ }
1254
+ ],
1255
+ "md5sum": "9be3a171e75708baaaf6249630cc7a0d"
1256
+ },
1257
+ {
1258
+ "dataPath": "params_shard_5.bin",
1259
+ "format": "raw-shard",
1260
+ "nbytes": 33076224,
1261
+ "records": [
1262
+ {
1263
+ "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.weight",
1264
+ "shape": [
1265
+ 3072,
1266
+ 768
1267
+ ],
1268
+ "dtype": "float16",
1269
+ "format": "f32-to-bf16",
1270
+ "nbytes": 4718592,
1271
+ "byteOffset": 0
1272
+ },
1273
+ {
1274
+ "name": "gpt_neox.layers.9.mlp.dense_h_to_4h.bias",
1275
+ "shape": [
1276
+ 3072
1277
+ ],
1278
+ "dtype": "float32",
1279
+ "format": "f32-to-bf16",
1280
+ "nbytes": 6144,
1281
+ "byteOffset": 4718592
1282
+ },
1283
+ {
1284
+ "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.weight",
1285
+ "shape": [
1286
+ 768,
1287
+ 3072
1288
+ ],
1289
+ "dtype": "float16",
1290
+ "format": "f32-to-bf16",
1291
+ "nbytes": 4718592,
1292
+ "byteOffset": 4724736
1293
+ },
1294
+ {
1295
+ "name": "gpt_neox.layers.9.mlp.dense_4h_to_h.bias",
1296
+ "shape": [
1297
+ 768
1298
+ ],
1299
+ "dtype": "float32",
1300
+ "format": "f32-to-bf16",
1301
+ "nbytes": 1536,
1302
+ "byteOffset": 9443328
1303
+ },
1304
+ {
1305
+ "name": "gpt_neox.layers.10.input_layernorm.weight",
1306
+ "shape": [
1307
+ 768
1308
+ ],
1309
+ "dtype": "float16",
1310
+ "format": "f32-to-bf16",
1311
+ "nbytes": 1536,
1312
+ "byteOffset": 9444864
1313
+ },
1314
+ {
1315
+ "name": "gpt_neox.layers.10.input_layernorm.bias",
1316
+ "shape": [
1317
+ 768
1318
+ ],
1319
+ "dtype": "float16",
1320
+ "format": "f32-to-bf16",
1321
+ "nbytes": 1536,
1322
+ "byteOffset": 9446400
1323
+ },
1324
+ {
1325
+ "name": "gpt_neox.layers.10.post_attention_layernorm.weight",
1326
+ "shape": [
1327
+ 768
1328
+ ],
1329
+ "dtype": "float16",
1330
+ "format": "f32-to-bf16",
1331
+ "nbytes": 1536,
1332
+ "byteOffset": 9447936
1333
+ },
1334
+ {
1335
+ "name": "gpt_neox.layers.10.post_attention_layernorm.bias",
1336
+ "shape": [
1337
+ 768
1338
+ ],
1339
+ "dtype": "float16",
1340
+ "format": "f32-to-bf16",
1341
+ "nbytes": 1536,
1342
+ "byteOffset": 9449472
1343
+ },
1344
+ {
1345
+ "name": "gpt_neox.layers.10.attention.query_key_value.weight",
1346
+ "shape": [
1347
+ 2304,
1348
+ 768
1349
+ ],
1350
+ "dtype": "float16",
1351
+ "format": "f32-to-bf16",
1352
+ "nbytes": 3538944,
1353
+ "byteOffset": 9451008
1354
+ },
1355
+ {
1356
+ "name": "gpt_neox.layers.10.attention.query_key_value.bias",
1357
+ "shape": [
1358
+ 2304
1359
+ ],
1360
+ "dtype": "float16",
1361
+ "format": "f32-to-bf16",
1362
+ "nbytes": 4608,
1363
+ "byteOffset": 12989952
1364
+ },
1365
+ {
1366
+ "name": "gpt_neox.layers.10.attention.dense.weight",
1367
+ "shape": [
1368
+ 768,
1369
+ 768
1370
+ ],
1371
+ "dtype": "float16",
1372
+ "format": "f32-to-bf16",
1373
+ "nbytes": 1179648,
1374
+ "byteOffset": 12994560
1375
+ },
1376
+ {
1377
+ "name": "gpt_neox.layers.10.attention.dense.bias",
1378
+ "shape": [
1379
+ 768
1380
+ ],
1381
+ "dtype": "float16",
1382
+ "format": "f32-to-bf16",
1383
+ "nbytes": 1536,
1384
+ "byteOffset": 14174208
1385
+ },
1386
+ {
1387
+ "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.weight",
1388
+ "shape": [
1389
+ 3072,
1390
+ 768
1391
+ ],
1392
+ "dtype": "float16",
1393
+ "format": "f32-to-bf16",
1394
+ "nbytes": 4718592,
1395
+ "byteOffset": 14175744
1396
+ },
1397
+ {
1398
+ "name": "gpt_neox.layers.10.mlp.dense_h_to_4h.bias",
1399
+ "shape": [
1400
+ 3072
1401
+ ],
1402
+ "dtype": "float32",
1403
+ "format": "f32-to-bf16",
1404
+ "nbytes": 6144,
1405
+ "byteOffset": 18894336
1406
+ },
1407
+ {
1408
+ "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.weight",
1409
+ "shape": [
1410
+ 768,
1411
+ 3072
1412
+ ],
1413
+ "dtype": "float16",
1414
+ "format": "f32-to-bf16",
1415
+ "nbytes": 4718592,
1416
+ "byteOffset": 18900480
1417
+ },
1418
+ {
1419
+ "name": "gpt_neox.layers.10.mlp.dense_4h_to_h.bias",
1420
+ "shape": [
1421
+ 768
1422
+ ],
1423
+ "dtype": "float32",
1424
+ "format": "f32-to-bf16",
1425
+ "nbytes": 1536,
1426
+ "byteOffset": 23619072
1427
+ },
1428
+ {
1429
+ "name": "gpt_neox.layers.11.input_layernorm.weight",
1430
+ "shape": [
1431
+ 768
1432
+ ],
1433
+ "dtype": "float16",
1434
+ "format": "f32-to-bf16",
1435
+ "nbytes": 1536,
1436
+ "byteOffset": 23620608
1437
+ },
1438
+ {
1439
+ "name": "gpt_neox.layers.11.input_layernorm.bias",
1440
+ "shape": [
1441
+ 768
1442
+ ],
1443
+ "dtype": "float16",
1444
+ "format": "f32-to-bf16",
1445
+ "nbytes": 1536,
1446
+ "byteOffset": 23622144
1447
+ },
1448
+ {
1449
+ "name": "gpt_neox.layers.11.post_attention_layernorm.weight",
1450
+ "shape": [
1451
+ 768
1452
+ ],
1453
+ "dtype": "float16",
1454
+ "format": "f32-to-bf16",
1455
+ "nbytes": 1536,
1456
+ "byteOffset": 23623680
1457
+ },
1458
+ {
1459
+ "name": "gpt_neox.layers.11.post_attention_layernorm.bias",
1460
+ "shape": [
1461
+ 768
1462
+ ],
1463
+ "dtype": "float16",
1464
+ "format": "f32-to-bf16",
1465
+ "nbytes": 1536,
1466
+ "byteOffset": 23625216
1467
+ },
1468
+ {
1469
+ "name": "gpt_neox.layers.11.attention.query_key_value.weight",
1470
+ "shape": [
1471
+ 2304,
1472
+ 768
1473
+ ],
1474
+ "dtype": "float16",
1475
+ "format": "f32-to-bf16",
1476
+ "nbytes": 3538944,
1477
+ "byteOffset": 23626752
1478
+ },
1479
+ {
1480
+ "name": "gpt_neox.layers.11.attention.query_key_value.bias",
1481
+ "shape": [
1482
+ 2304
1483
+ ],
1484
+ "dtype": "float16",
1485
+ "format": "f32-to-bf16",
1486
+ "nbytes": 4608,
1487
+ "byteOffset": 27165696
1488
+ },
1489
+ {
1490
+ "name": "gpt_neox.layers.11.attention.dense.weight",
1491
+ "shape": [
1492
+ 768,
1493
+ 768
1494
+ ],
1495
+ "dtype": "float16",
1496
+ "format": "f32-to-bf16",
1497
+ "nbytes": 1179648,
1498
+ "byteOffset": 27170304
1499
+ },
1500
+ {
1501
+ "name": "gpt_neox.layers.11.attention.dense.bias",
1502
+ "shape": [
1503
+ 768
1504
+ ],
1505
+ "dtype": "float16",
1506
+ "format": "f32-to-bf16",
1507
+ "nbytes": 1536,
1508
+ "byteOffset": 28349952
1509
+ },
1510
+ {
1511
+ "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.weight",
1512
+ "shape": [
1513
+ 3072,
1514
+ 768
1515
+ ],
1516
+ "dtype": "float16",
1517
+ "format": "f32-to-bf16",
1518
+ "nbytes": 4718592,
1519
+ "byteOffset": 28351488
1520
+ },
1521
+ {
1522
+ "name": "gpt_neox.layers.11.mlp.dense_h_to_4h.bias",
1523
+ "shape": [
1524
+ 3072
1525
+ ],
1526
+ "dtype": "float32",
1527
+ "format": "f32-to-bf16",
1528
+ "nbytes": 6144,
1529
+ "byteOffset": 33070080
1530
+ }
1531
+ ],
1532
+ "md5sum": "d3807c10c1e4cf1ed703d786dad97dd5"
1533
+ },
1534
+ {
1535
+ "dataPath": "params_shard_6.bin",
1536
+ "format": "raw-shard",
1537
+ "nbytes": 77266944,
1538
+ "records": [
1539
+ {
1540
+ "name": "embed_out.weight",
1541
+ "shape": [
1542
+ 50304,
1543
+ 768
1544
+ ],
1545
+ "dtype": "float16",
1546
+ "format": "f32-to-bf16",
1547
+ "nbytes": 77266944,
1548
+ "byteOffset": 0
1549
+ }
1550
+ ],
1551
+ "md5sum": "084dbad7777340578a2501ea88f226dd"
1552
+ },
1553
+ {
1554
+ "dataPath": "params_shard_7.bin",
1555
+ "format": "raw-shard",
1556
+ "nbytes": 4723200,
1557
+ "records": [
1558
+ {
1559
+ "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.weight",
1560
+ "shape": [
1561
+ 768,
1562
+ 3072
1563
+ ],
1564
+ "dtype": "float16",
1565
+ "format": "f32-to-bf16",
1566
+ "nbytes": 4718592,
1567
+ "byteOffset": 0
1568
+ },
1569
+ {
1570
+ "name": "gpt_neox.layers.11.mlp.dense_4h_to_h.bias",
1571
+ "shape": [
1572
+ 768
1573
+ ],
1574
+ "dtype": "float32",
1575
+ "format": "f32-to-bf16",
1576
+ "nbytes": 1536,
1577
+ "byteOffset": 4718592
1578
+ },
1579
+ {
1580
+ "name": "gpt_neox.final_layer_norm.weight",
1581
+ "shape": [
1582
+ 768
1583
+ ],
1584
+ "dtype": "float16",
1585
+ "format": "f32-to-bf16",
1586
+ "nbytes": 1536,
1587
+ "byteOffset": 4720128
1588
+ },
1589
+ {
1590
+ "name": "gpt_neox.final_layer_norm.bias",
1591
+ "shape": [
1592
+ 768
1593
+ ],
1594
+ "dtype": "float16",
1595
+ "format": "f32-to-bf16",
1596
+ "nbytes": 1536,
1597
+ "byteOffset": 4721664
1598
+ }
1599
+ ],
1600
+ "md5sum": "5290d766289dc0d43e7407c8dc728b2e"
1601
+ }
1602
+ ]
1603
+ }
params_shard_0.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:614b66a3894ca0ab28b37ba3b3e59e62b3fd52d128380161c7064ff7e2304d77
3
+ size 77266944
params_shard_1.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b70ba3debe1ded96f69c9ddf5ba9529505140257b3bc9e8ce01215e2510e7a22
3
+ size 33082368
params_shard_2.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4e827fcafb1edf60279b0520d28b77f273bb653349d5b6b0d526cb562906d6c
3
+ size 33076224
params_shard_3.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b68bdfe2993286089ac67f25c5779bc9c84671d1a29a78c4fd9fd26e5dd7eecb
3
+ size 33077760
params_shard_4.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e2d37c4afd8fb00776c8c0763f72c484010ff7737f74676b642832be8d340848
3
+ size 33076224
params_shard_5.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a7a39322516213617081e7693fc2c533d53072ab26b0841ae54d119f74833086
3
+ size 33076224
params_shard_6.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0cf4456991f3d8a515b0c34301d30931acfb37e6025f9e499f0c2cef5e409dfe
3
+ size 77266944
params_shard_7.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ffa9560caed6adb2e39e1854caa303c5eefbe16e97a3677acae16866be567861
3
+ size 4723200
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": "<|endoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "name_or_path": "EleutherAI/gpt-neox-20b",
6
+ "special_tokens_map_file": "/admin/home-hailey/.cache/huggingface/hub/models--EleutherAI--gpt-neox-20b/snapshots/4e49eadb5d14bd22f314ec3f45b69a87b88c7691/special_tokens_map.json",
7
+ "tokenizer_class": "GPTNeoXTokenizer",
8
+ "unk_token": "<|endoftext|>"
9
+ }