Amai commited on
Commit
cb68c41
·
1 Parent(s): fd48e4d

Add weights

Browse files
EleutherAI_pythia-70m-android-q0f16.tar ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a1defcce5ade4a339bc0e3c7abca4803ad25398814df77181db8d1a7af450a0c
3
+ size 294031
ir_dump/debug-final.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase0.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase1.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase2.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase3.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase4.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/debug-phase5.py ADDED
The diff for this file is too large to render. See raw diff
 
ir_dump/kernel.cl ADDED
The diff for this file is too large to render. See raw diff
 
mlc-chat-config.json ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "0.1.0",
3
+ "model_type": "gpt_neox",
4
+ "quantization": "q0f16",
5
+ "model_config": {
6
+ "use_parallel_residual": true,
7
+ "hidden_size": 512,
8
+ "intermediate_size": 2048,
9
+ "num_attention_heads": 8,
10
+ "num_hidden_layers": 6,
11
+ "layer_norm_eps": 1e-05,
12
+ "vocab_size": 50304,
13
+ "rotary_pct": 0.25,
14
+ "position_embedding_base": 10000,
15
+ "context_window_size": 2048,
16
+ "head_dim": 64,
17
+ "prefill_chunk_size": 2048,
18
+ "tensor_parallel_shards": 1,
19
+ "ffn_out_dtype": "float32",
20
+ "max_batch_size": 128
21
+ },
22
+ "vocab_size": 50304,
23
+ "context_window_size": 2048,
24
+ "sliding_window_size": -1,
25
+ "prefill_chunk_size": 2048,
26
+ "attention_sink_size": -1,
27
+ "tensor_parallel_shards": 1,
28
+ "pipeline_parallel_stages": 1,
29
+ "temperature": 1.0,
30
+ "presence_penalty": 0.0,
31
+ "frequency_penalty": 0.0,
32
+ "repetition_penalty": 1.0,
33
+ "top_p": 1.0,
34
+ "tokenizer_files": [
35
+ "tokenizer.json",
36
+ "tokenizer_config.json"
37
+ ],
38
+ "tokenizer_info": {
39
+ "token_postproc_method": "byte_level",
40
+ "prepend_space_in_encode": false,
41
+ "strip_space_in_decode": false
42
+ },
43
+ "conv_template": {
44
+ "name": "gpt2",
45
+ "system_template": "{system_message}",
46
+ "system_message": "",
47
+ "system_prefix_token_ids": null,
48
+ "add_role_after_system_message": true,
49
+ "roles": {
50
+ "user": "",
51
+ "assistant": ""
52
+ },
53
+ "role_templates": {
54
+ "user": "{user_message}",
55
+ "assistant": "{assistant_message}",
56
+ "tool": "{tool_message}"
57
+ },
58
+ "messages": [],
59
+ "seps": [
60
+ ""
61
+ ],
62
+ "role_content_sep": "",
63
+ "role_empty_sep": "",
64
+ "stop_str": [
65
+ "</s>"
66
+ ],
67
+ "stop_token_ids": [
68
+ 50256
69
+ ],
70
+ "function_string": "",
71
+ "use_function_calling": false
72
+ },
73
+ "pad_token_id": 0,
74
+ "bos_token_id": 0,
75
+ "eos_token_id": 0
76
+ }
ndarray-cache-b16.json ADDED
@@ -0,0 +1,827 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "ParamSize": 76,
4
+ "ParamBytes": 140883968.0,
5
+ "BitsPerParam": 16.00348958939165
6
+ },
7
+ "records": [
8
+ {
9
+ "dataPath": "params_shard_0.bin",
10
+ "format": "raw-shard",
11
+ "nbytes": 51511296,
12
+ "records": [
13
+ {
14
+ "name": "gpt_neox.embed_in.weight",
15
+ "shape": [
16
+ 50304,
17
+ 512
18
+ ],
19
+ "dtype": "float16",
20
+ "format": "f32-to-bf16",
21
+ "nbytes": 51511296,
22
+ "byteOffset": 0
23
+ }
24
+ ],
25
+ "md5sum": "a1619d49d0b0ba376d0f671d12c13e92"
26
+ },
27
+ {
28
+ "dataPath": "params_shard_1.bin",
29
+ "format": "raw-shard",
30
+ "nbytes": 33103872,
31
+ "records": [
32
+ {
33
+ "name": "gpt_neox.layers.0.input_layernorm.weight",
34
+ "shape": [
35
+ 512
36
+ ],
37
+ "dtype": "float16",
38
+ "format": "f32-to-bf16",
39
+ "nbytes": 1024,
40
+ "byteOffset": 0
41
+ },
42
+ {
43
+ "name": "gpt_neox.layers.0.input_layernorm.bias",
44
+ "shape": [
45
+ 512
46
+ ],
47
+ "dtype": "float16",
48
+ "format": "f32-to-bf16",
49
+ "nbytes": 1024,
50
+ "byteOffset": 1024
51
+ },
52
+ {
53
+ "name": "gpt_neox.layers.0.post_attention_layernorm.weight",
54
+ "shape": [
55
+ 512
56
+ ],
57
+ "dtype": "float16",
58
+ "format": "f32-to-bf16",
59
+ "nbytes": 1024,
60
+ "byteOffset": 2048
61
+ },
62
+ {
63
+ "name": "gpt_neox.layers.0.post_attention_layernorm.bias",
64
+ "shape": [
65
+ 512
66
+ ],
67
+ "dtype": "float16",
68
+ "format": "f32-to-bf16",
69
+ "nbytes": 1024,
70
+ "byteOffset": 3072
71
+ },
72
+ {
73
+ "name": "gpt_neox.layers.0.attention.query_key_value.weight",
74
+ "shape": [
75
+ 1536,
76
+ 512
77
+ ],
78
+ "dtype": "float16",
79
+ "format": "f32-to-bf16",
80
+ "nbytes": 1572864,
81
+ "byteOffset": 4096
82
+ },
83
+ {
84
+ "name": "gpt_neox.layers.0.attention.query_key_value.bias",
85
+ "shape": [
86
+ 1536
87
+ ],
88
+ "dtype": "float16",
89
+ "format": "f32-to-bf16",
90
+ "nbytes": 3072,
91
+ "byteOffset": 1576960
92
+ },
93
+ {
94
+ "name": "gpt_neox.layers.0.attention.dense.weight",
95
+ "shape": [
96
+ 512,
97
+ 512
98
+ ],
99
+ "dtype": "float16",
100
+ "format": "f32-to-bf16",
101
+ "nbytes": 524288,
102
+ "byteOffset": 1580032
103
+ },
104
+ {
105
+ "name": "gpt_neox.layers.0.attention.dense.bias",
106
+ "shape": [
107
+ 512
108
+ ],
109
+ "dtype": "float16",
110
+ "format": "f32-to-bf16",
111
+ "nbytes": 1024,
112
+ "byteOffset": 2104320
113
+ },
114
+ {
115
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight",
116
+ "shape": [
117
+ 2048,
118
+ 512
119
+ ],
120
+ "dtype": "float16",
121
+ "format": "f32-to-bf16",
122
+ "nbytes": 2097152,
123
+ "byteOffset": 2105344
124
+ },
125
+ {
126
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
127
+ "shape": [
128
+ 2048
129
+ ],
130
+ "dtype": "bfloat16",
131
+ "format": "raw",
132
+ "nbytes": 4096,
133
+ "byteOffset": 4202496
134
+ },
135
+ {
136
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight",
137
+ "shape": [
138
+ 512,
139
+ 2048
140
+ ],
141
+ "dtype": "float16",
142
+ "format": "f32-to-bf16",
143
+ "nbytes": 2097152,
144
+ "byteOffset": 4206592
145
+ },
146
+ {
147
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
148
+ "shape": [
149
+ 512
150
+ ],
151
+ "dtype": "bfloat16",
152
+ "format": "raw",
153
+ "nbytes": 1024,
154
+ "byteOffset": 6303744
155
+ },
156
+ {
157
+ "name": "gpt_neox.layers.1.input_layernorm.weight",
158
+ "shape": [
159
+ 512
160
+ ],
161
+ "dtype": "float16",
162
+ "format": "f32-to-bf16",
163
+ "nbytes": 1024,
164
+ "byteOffset": 6304768
165
+ },
166
+ {
167
+ "name": "gpt_neox.layers.1.input_layernorm.bias",
168
+ "shape": [
169
+ 512
170
+ ],
171
+ "dtype": "float16",
172
+ "format": "f32-to-bf16",
173
+ "nbytes": 1024,
174
+ "byteOffset": 6305792
175
+ },
176
+ {
177
+ "name": "gpt_neox.layers.1.post_attention_layernorm.weight",
178
+ "shape": [
179
+ 512
180
+ ],
181
+ "dtype": "float16",
182
+ "format": "f32-to-bf16",
183
+ "nbytes": 1024,
184
+ "byteOffset": 6306816
185
+ },
186
+ {
187
+ "name": "gpt_neox.layers.1.post_attention_layernorm.bias",
188
+ "shape": [
189
+ 512
190
+ ],
191
+ "dtype": "float16",
192
+ "format": "f32-to-bf16",
193
+ "nbytes": 1024,
194
+ "byteOffset": 6307840
195
+ },
196
+ {
197
+ "name": "gpt_neox.layers.1.attention.query_key_value.weight",
198
+ "shape": [
199
+ 1536,
200
+ 512
201
+ ],
202
+ "dtype": "float16",
203
+ "format": "f32-to-bf16",
204
+ "nbytes": 1572864,
205
+ "byteOffset": 6308864
206
+ },
207
+ {
208
+ "name": "gpt_neox.layers.1.attention.query_key_value.bias",
209
+ "shape": [
210
+ 1536
211
+ ],
212
+ "dtype": "float16",
213
+ "format": "f32-to-bf16",
214
+ "nbytes": 3072,
215
+ "byteOffset": 7881728
216
+ },
217
+ {
218
+ "name": "gpt_neox.layers.1.attention.dense.weight",
219
+ "shape": [
220
+ 512,
221
+ 512
222
+ ],
223
+ "dtype": "float16",
224
+ "format": "f32-to-bf16",
225
+ "nbytes": 524288,
226
+ "byteOffset": 7884800
227
+ },
228
+ {
229
+ "name": "gpt_neox.layers.1.attention.dense.bias",
230
+ "shape": [
231
+ 512
232
+ ],
233
+ "dtype": "float16",
234
+ "format": "f32-to-bf16",
235
+ "nbytes": 1024,
236
+ "byteOffset": 8409088
237
+ },
238
+ {
239
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight",
240
+ "shape": [
241
+ 2048,
242
+ 512
243
+ ],
244
+ "dtype": "float16",
245
+ "format": "f32-to-bf16",
246
+ "nbytes": 2097152,
247
+ "byteOffset": 8410112
248
+ },
249
+ {
250
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
251
+ "shape": [
252
+ 2048
253
+ ],
254
+ "dtype": "bfloat16",
255
+ "format": "raw",
256
+ "nbytes": 4096,
257
+ "byteOffset": 10507264
258
+ },
259
+ {
260
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight",
261
+ "shape": [
262
+ 512,
263
+ 2048
264
+ ],
265
+ "dtype": "float16",
266
+ "format": "f32-to-bf16",
267
+ "nbytes": 2097152,
268
+ "byteOffset": 10511360
269
+ },
270
+ {
271
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
272
+ "shape": [
273
+ 512
274
+ ],
275
+ "dtype": "bfloat16",
276
+ "format": "raw",
277
+ "nbytes": 1024,
278
+ "byteOffset": 12608512
279
+ },
280
+ {
281
+ "name": "gpt_neox.layers.2.input_layernorm.weight",
282
+ "shape": [
283
+ 512
284
+ ],
285
+ "dtype": "float16",
286
+ "format": "f32-to-bf16",
287
+ "nbytes": 1024,
288
+ "byteOffset": 12609536
289
+ },
290
+ {
291
+ "name": "gpt_neox.layers.2.input_layernorm.bias",
292
+ "shape": [
293
+ 512
294
+ ],
295
+ "dtype": "float16",
296
+ "format": "f32-to-bf16",
297
+ "nbytes": 1024,
298
+ "byteOffset": 12610560
299
+ },
300
+ {
301
+ "name": "gpt_neox.layers.2.post_attention_layernorm.weight",
302
+ "shape": [
303
+ 512
304
+ ],
305
+ "dtype": "float16",
306
+ "format": "f32-to-bf16",
307
+ "nbytes": 1024,
308
+ "byteOffset": 12611584
309
+ },
310
+ {
311
+ "name": "gpt_neox.layers.2.post_attention_layernorm.bias",
312
+ "shape": [
313
+ 512
314
+ ],
315
+ "dtype": "float16",
316
+ "format": "f32-to-bf16",
317
+ "nbytes": 1024,
318
+ "byteOffset": 12612608
319
+ },
320
+ {
321
+ "name": "gpt_neox.layers.2.attention.query_key_value.weight",
322
+ "shape": [
323
+ 1536,
324
+ 512
325
+ ],
326
+ "dtype": "float16",
327
+ "format": "f32-to-bf16",
328
+ "nbytes": 1572864,
329
+ "byteOffset": 12613632
330
+ },
331
+ {
332
+ "name": "gpt_neox.layers.2.attention.query_key_value.bias",
333
+ "shape": [
334
+ 1536
335
+ ],
336
+ "dtype": "float16",
337
+ "format": "f32-to-bf16",
338
+ "nbytes": 3072,
339
+ "byteOffset": 14186496
340
+ },
341
+ {
342
+ "name": "gpt_neox.layers.2.attention.dense.weight",
343
+ "shape": [
344
+ 512,
345
+ 512
346
+ ],
347
+ "dtype": "float16",
348
+ "format": "f32-to-bf16",
349
+ "nbytes": 524288,
350
+ "byteOffset": 14189568
351
+ },
352
+ {
353
+ "name": "gpt_neox.layers.2.attention.dense.bias",
354
+ "shape": [
355
+ 512
356
+ ],
357
+ "dtype": "float16",
358
+ "format": "f32-to-bf16",
359
+ "nbytes": 1024,
360
+ "byteOffset": 14713856
361
+ },
362
+ {
363
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight",
364
+ "shape": [
365
+ 2048,
366
+ 512
367
+ ],
368
+ "dtype": "float16",
369
+ "format": "f32-to-bf16",
370
+ "nbytes": 2097152,
371
+ "byteOffset": 14714880
372
+ },
373
+ {
374
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
375
+ "shape": [
376
+ 2048
377
+ ],
378
+ "dtype": "bfloat16",
379
+ "format": "raw",
380
+ "nbytes": 4096,
381
+ "byteOffset": 16812032
382
+ },
383
+ {
384
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight",
385
+ "shape": [
386
+ 512,
387
+ 2048
388
+ ],
389
+ "dtype": "float16",
390
+ "format": "f32-to-bf16",
391
+ "nbytes": 2097152,
392
+ "byteOffset": 16816128
393
+ },
394
+ {
395
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
396
+ "shape": [
397
+ 512
398
+ ],
399
+ "dtype": "bfloat16",
400
+ "format": "raw",
401
+ "nbytes": 1024,
402
+ "byteOffset": 18913280
403
+ },
404
+ {
405
+ "name": "gpt_neox.layers.3.input_layernorm.weight",
406
+ "shape": [
407
+ 512
408
+ ],
409
+ "dtype": "float16",
410
+ "format": "f32-to-bf16",
411
+ "nbytes": 1024,
412
+ "byteOffset": 18914304
413
+ },
414
+ {
415
+ "name": "gpt_neox.layers.3.input_layernorm.bias",
416
+ "shape": [
417
+ 512
418
+ ],
419
+ "dtype": "float16",
420
+ "format": "f32-to-bf16",
421
+ "nbytes": 1024,
422
+ "byteOffset": 18915328
423
+ },
424
+ {
425
+ "name": "gpt_neox.layers.3.post_attention_layernorm.weight",
426
+ "shape": [
427
+ 512
428
+ ],
429
+ "dtype": "float16",
430
+ "format": "f32-to-bf16",
431
+ "nbytes": 1024,
432
+ "byteOffset": 18916352
433
+ },
434
+ {
435
+ "name": "gpt_neox.layers.3.post_attention_layernorm.bias",
436
+ "shape": [
437
+ 512
438
+ ],
439
+ "dtype": "float16",
440
+ "format": "f32-to-bf16",
441
+ "nbytes": 1024,
442
+ "byteOffset": 18917376
443
+ },
444
+ {
445
+ "name": "gpt_neox.layers.3.attention.query_key_value.weight",
446
+ "shape": [
447
+ 1536,
448
+ 512
449
+ ],
450
+ "dtype": "float16",
451
+ "format": "f32-to-bf16",
452
+ "nbytes": 1572864,
453
+ "byteOffset": 18918400
454
+ },
455
+ {
456
+ "name": "gpt_neox.layers.3.attention.query_key_value.bias",
457
+ "shape": [
458
+ 1536
459
+ ],
460
+ "dtype": "float16",
461
+ "format": "f32-to-bf16",
462
+ "nbytes": 3072,
463
+ "byteOffset": 20491264
464
+ },
465
+ {
466
+ "name": "gpt_neox.layers.3.attention.dense.weight",
467
+ "shape": [
468
+ 512,
469
+ 512
470
+ ],
471
+ "dtype": "float16",
472
+ "format": "f32-to-bf16",
473
+ "nbytes": 524288,
474
+ "byteOffset": 20494336
475
+ },
476
+ {
477
+ "name": "gpt_neox.layers.3.attention.dense.bias",
478
+ "shape": [
479
+ 512
480
+ ],
481
+ "dtype": "float16",
482
+ "format": "f32-to-bf16",
483
+ "nbytes": 1024,
484
+ "byteOffset": 21018624
485
+ },
486
+ {
487
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight",
488
+ "shape": [
489
+ 2048,
490
+ 512
491
+ ],
492
+ "dtype": "float16",
493
+ "format": "f32-to-bf16",
494
+ "nbytes": 2097152,
495
+ "byteOffset": 21019648
496
+ },
497
+ {
498
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
499
+ "shape": [
500
+ 2048
501
+ ],
502
+ "dtype": "bfloat16",
503
+ "format": "raw",
504
+ "nbytes": 4096,
505
+ "byteOffset": 23116800
506
+ },
507
+ {
508
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight",
509
+ "shape": [
510
+ 512,
511
+ 2048
512
+ ],
513
+ "dtype": "float16",
514
+ "format": "f32-to-bf16",
515
+ "nbytes": 2097152,
516
+ "byteOffset": 23120896
517
+ },
518
+ {
519
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
520
+ "shape": [
521
+ 512
522
+ ],
523
+ "dtype": "bfloat16",
524
+ "format": "raw",
525
+ "nbytes": 1024,
526
+ "byteOffset": 25218048
527
+ },
528
+ {
529
+ "name": "gpt_neox.layers.4.input_layernorm.weight",
530
+ "shape": [
531
+ 512
532
+ ],
533
+ "dtype": "float16",
534
+ "format": "f32-to-bf16",
535
+ "nbytes": 1024,
536
+ "byteOffset": 25219072
537
+ },
538
+ {
539
+ "name": "gpt_neox.layers.4.input_layernorm.bias",
540
+ "shape": [
541
+ 512
542
+ ],
543
+ "dtype": "float16",
544
+ "format": "f32-to-bf16",
545
+ "nbytes": 1024,
546
+ "byteOffset": 25220096
547
+ },
548
+ {
549
+ "name": "gpt_neox.layers.4.post_attention_layernorm.weight",
550
+ "shape": [
551
+ 512
552
+ ],
553
+ "dtype": "float16",
554
+ "format": "f32-to-bf16",
555
+ "nbytes": 1024,
556
+ "byteOffset": 25221120
557
+ },
558
+ {
559
+ "name": "gpt_neox.layers.4.post_attention_layernorm.bias",
560
+ "shape": [
561
+ 512
562
+ ],
563
+ "dtype": "float16",
564
+ "format": "f32-to-bf16",
565
+ "nbytes": 1024,
566
+ "byteOffset": 25222144
567
+ },
568
+ {
569
+ "name": "gpt_neox.layers.4.attention.query_key_value.weight",
570
+ "shape": [
571
+ 1536,
572
+ 512
573
+ ],
574
+ "dtype": "float16",
575
+ "format": "f32-to-bf16",
576
+ "nbytes": 1572864,
577
+ "byteOffset": 25223168
578
+ },
579
+ {
580
+ "name": "gpt_neox.layers.4.attention.query_key_value.bias",
581
+ "shape": [
582
+ 1536
583
+ ],
584
+ "dtype": "float16",
585
+ "format": "f32-to-bf16",
586
+ "nbytes": 3072,
587
+ "byteOffset": 26796032
588
+ },
589
+ {
590
+ "name": "gpt_neox.layers.4.attention.dense.weight",
591
+ "shape": [
592
+ 512,
593
+ 512
594
+ ],
595
+ "dtype": "float16",
596
+ "format": "f32-to-bf16",
597
+ "nbytes": 524288,
598
+ "byteOffset": 26799104
599
+ },
600
+ {
601
+ "name": "gpt_neox.layers.4.attention.dense.bias",
602
+ "shape": [
603
+ 512
604
+ ],
605
+ "dtype": "float16",
606
+ "format": "f32-to-bf16",
607
+ "nbytes": 1024,
608
+ "byteOffset": 27323392
609
+ },
610
+ {
611
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight",
612
+ "shape": [
613
+ 2048,
614
+ 512
615
+ ],
616
+ "dtype": "float16",
617
+ "format": "f32-to-bf16",
618
+ "nbytes": 2097152,
619
+ "byteOffset": 27324416
620
+ },
621
+ {
622
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
623
+ "shape": [
624
+ 2048
625
+ ],
626
+ "dtype": "bfloat16",
627
+ "format": "raw",
628
+ "nbytes": 4096,
629
+ "byteOffset": 29421568
630
+ },
631
+ {
632
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight",
633
+ "shape": [
634
+ 512,
635
+ 2048
636
+ ],
637
+ "dtype": "float16",
638
+ "format": "f32-to-bf16",
639
+ "nbytes": 2097152,
640
+ "byteOffset": 29425664
641
+ },
642
+ {
643
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
644
+ "shape": [
645
+ 512
646
+ ],
647
+ "dtype": "bfloat16",
648
+ "format": "raw",
649
+ "nbytes": 1024,
650
+ "byteOffset": 31522816
651
+ },
652
+ {
653
+ "name": "gpt_neox.layers.5.input_layernorm.weight",
654
+ "shape": [
655
+ 512
656
+ ],
657
+ "dtype": "float16",
658
+ "format": "f32-to-bf16",
659
+ "nbytes": 1024,
660
+ "byteOffset": 31523840
661
+ },
662
+ {
663
+ "name": "gpt_neox.layers.5.input_layernorm.bias",
664
+ "shape": [
665
+ 512
666
+ ],
667
+ "dtype": "float16",
668
+ "format": "f32-to-bf16",
669
+ "nbytes": 1024,
670
+ "byteOffset": 31524864
671
+ },
672
+ {
673
+ "name": "gpt_neox.layers.5.post_attention_layernorm.weight",
674
+ "shape": [
675
+ 512
676
+ ],
677
+ "dtype": "float16",
678
+ "format": "f32-to-bf16",
679
+ "nbytes": 1024,
680
+ "byteOffset": 31525888
681
+ },
682
+ {
683
+ "name": "gpt_neox.layers.5.post_attention_layernorm.bias",
684
+ "shape": [
685
+ 512
686
+ ],
687
+ "dtype": "float16",
688
+ "format": "f32-to-bf16",
689
+ "nbytes": 1024,
690
+ "byteOffset": 31526912
691
+ },
692
+ {
693
+ "name": "gpt_neox.layers.5.attention.query_key_value.weight",
694
+ "shape": [
695
+ 1536,
696
+ 512
697
+ ],
698
+ "dtype": "float16",
699
+ "format": "f32-to-bf16",
700
+ "nbytes": 1572864,
701
+ "byteOffset": 31527936
702
+ },
703
+ {
704
+ "name": "gpt_neox.layers.5.attention.query_key_value.bias",
705
+ "shape": [
706
+ 1536
707
+ ],
708
+ "dtype": "float16",
709
+ "format": "f32-to-bf16",
710
+ "nbytes": 3072,
711
+ "byteOffset": 33100800
712
+ }
713
+ ],
714
+ "md5sum": "ef146e3d692ff0f92b4887051f5ce1aa"
715
+ },
716
+ {
717
+ "dataPath": "params_shard_2.bin",
718
+ "format": "raw-shard",
719
+ "nbytes": 51511296,
720
+ "records": [
721
+ {
722
+ "name": "embed_out.weight",
723
+ "shape": [
724
+ 50304,
725
+ 512
726
+ ],
727
+ "dtype": "float16",
728
+ "format": "f32-to-bf16",
729
+ "nbytes": 51511296,
730
+ "byteOffset": 0
731
+ }
732
+ ],
733
+ "md5sum": "2f016b975538d1e27f57fc84596c2432"
734
+ },
735
+ {
736
+ "dataPath": "params_shard_3.bin",
737
+ "format": "raw-shard",
738
+ "nbytes": 4726784,
739
+ "records": [
740
+ {
741
+ "name": "gpt_neox.layers.5.attention.dense.weight",
742
+ "shape": [
743
+ 512,
744
+ 512
745
+ ],
746
+ "dtype": "float16",
747
+ "format": "f32-to-bf16",
748
+ "nbytes": 524288,
749
+ "byteOffset": 0
750
+ },
751
+ {
752
+ "name": "gpt_neox.layers.5.attention.dense.bias",
753
+ "shape": [
754
+ 512
755
+ ],
756
+ "dtype": "float16",
757
+ "format": "f32-to-bf16",
758
+ "nbytes": 1024,
759
+ "byteOffset": 524288
760
+ },
761
+ {
762
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight",
763
+ "shape": [
764
+ 2048,
765
+ 512
766
+ ],
767
+ "dtype": "float16",
768
+ "format": "f32-to-bf16",
769
+ "nbytes": 2097152,
770
+ "byteOffset": 525312
771
+ },
772
+ {
773
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
774
+ "shape": [
775
+ 2048
776
+ ],
777
+ "dtype": "bfloat16",
778
+ "format": "raw",
779
+ "nbytes": 4096,
780
+ "byteOffset": 2622464
781
+ },
782
+ {
783
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight",
784
+ "shape": [
785
+ 512,
786
+ 2048
787
+ ],
788
+ "dtype": "float16",
789
+ "format": "f32-to-bf16",
790
+ "nbytes": 2097152,
791
+ "byteOffset": 2626560
792
+ },
793
+ {
794
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
795
+ "shape": [
796
+ 512
797
+ ],
798
+ "dtype": "bfloat16",
799
+ "format": "raw",
800
+ "nbytes": 1024,
801
+ "byteOffset": 4723712
802
+ },
803
+ {
804
+ "name": "gpt_neox.final_layer_norm.weight",
805
+ "shape": [
806
+ 512
807
+ ],
808
+ "dtype": "float16",
809
+ "format": "f32-to-bf16",
810
+ "nbytes": 1024,
811
+ "byteOffset": 4724736
812
+ },
813
+ {
814
+ "name": "gpt_neox.final_layer_norm.bias",
815
+ "shape": [
816
+ 512
817
+ ],
818
+ "dtype": "float16",
819
+ "format": "f32-to-bf16",
820
+ "nbytes": 1024,
821
+ "byteOffset": 4725760
822
+ }
823
+ ],
824
+ "md5sum": "4b204eca2568a39d4f462b042c0e9ab7"
825
+ }
826
+ ]
827
+ }
ndarray-cache.json ADDED
@@ -0,0 +1,827 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "ParamSize": 76,
4
+ "ParamBytes": 140883968.0,
5
+ "BitsPerParam": 16.00348958939165
6
+ },
7
+ "records": [
8
+ {
9
+ "dataPath": "params_shard_0.bin",
10
+ "format": "raw-shard",
11
+ "nbytes": 51511296,
12
+ "records": [
13
+ {
14
+ "name": "gpt_neox.embed_in.weight",
15
+ "shape": [
16
+ 50304,
17
+ 512
18
+ ],
19
+ "dtype": "float16",
20
+ "format": "f32-to-bf16",
21
+ "nbytes": 51511296,
22
+ "byteOffset": 0
23
+ }
24
+ ],
25
+ "md5sum": "a1619d49d0b0ba376d0f671d12c13e92"
26
+ },
27
+ {
28
+ "dataPath": "params_shard_1.bin",
29
+ "format": "raw-shard",
30
+ "nbytes": 33103872,
31
+ "records": [
32
+ {
33
+ "name": "gpt_neox.layers.0.input_layernorm.weight",
34
+ "shape": [
35
+ 512
36
+ ],
37
+ "dtype": "float16",
38
+ "format": "f32-to-bf16",
39
+ "nbytes": 1024,
40
+ "byteOffset": 0
41
+ },
42
+ {
43
+ "name": "gpt_neox.layers.0.input_layernorm.bias",
44
+ "shape": [
45
+ 512
46
+ ],
47
+ "dtype": "float16",
48
+ "format": "f32-to-bf16",
49
+ "nbytes": 1024,
50
+ "byteOffset": 1024
51
+ },
52
+ {
53
+ "name": "gpt_neox.layers.0.post_attention_layernorm.weight",
54
+ "shape": [
55
+ 512
56
+ ],
57
+ "dtype": "float16",
58
+ "format": "f32-to-bf16",
59
+ "nbytes": 1024,
60
+ "byteOffset": 2048
61
+ },
62
+ {
63
+ "name": "gpt_neox.layers.0.post_attention_layernorm.bias",
64
+ "shape": [
65
+ 512
66
+ ],
67
+ "dtype": "float16",
68
+ "format": "f32-to-bf16",
69
+ "nbytes": 1024,
70
+ "byteOffset": 3072
71
+ },
72
+ {
73
+ "name": "gpt_neox.layers.0.attention.query_key_value.weight",
74
+ "shape": [
75
+ 1536,
76
+ 512
77
+ ],
78
+ "dtype": "float16",
79
+ "format": "f32-to-bf16",
80
+ "nbytes": 1572864,
81
+ "byteOffset": 4096
82
+ },
83
+ {
84
+ "name": "gpt_neox.layers.0.attention.query_key_value.bias",
85
+ "shape": [
86
+ 1536
87
+ ],
88
+ "dtype": "float16",
89
+ "format": "f32-to-bf16",
90
+ "nbytes": 3072,
91
+ "byteOffset": 1576960
92
+ },
93
+ {
94
+ "name": "gpt_neox.layers.0.attention.dense.weight",
95
+ "shape": [
96
+ 512,
97
+ 512
98
+ ],
99
+ "dtype": "float16",
100
+ "format": "f32-to-bf16",
101
+ "nbytes": 524288,
102
+ "byteOffset": 1580032
103
+ },
104
+ {
105
+ "name": "gpt_neox.layers.0.attention.dense.bias",
106
+ "shape": [
107
+ 512
108
+ ],
109
+ "dtype": "float16",
110
+ "format": "f32-to-bf16",
111
+ "nbytes": 1024,
112
+ "byteOffset": 2104320
113
+ },
114
+ {
115
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.weight",
116
+ "shape": [
117
+ 2048,
118
+ 512
119
+ ],
120
+ "dtype": "float16",
121
+ "format": "f32-to-bf16",
122
+ "nbytes": 2097152,
123
+ "byteOffset": 2105344
124
+ },
125
+ {
126
+ "name": "gpt_neox.layers.0.mlp.dense_h_to_4h.bias",
127
+ "shape": [
128
+ 2048
129
+ ],
130
+ "dtype": "float32",
131
+ "format": "f32-to-bf16",
132
+ "nbytes": 4096,
133
+ "byteOffset": 4202496
134
+ },
135
+ {
136
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.weight",
137
+ "shape": [
138
+ 512,
139
+ 2048
140
+ ],
141
+ "dtype": "float16",
142
+ "format": "f32-to-bf16",
143
+ "nbytes": 2097152,
144
+ "byteOffset": 4206592
145
+ },
146
+ {
147
+ "name": "gpt_neox.layers.0.mlp.dense_4h_to_h.bias",
148
+ "shape": [
149
+ 512
150
+ ],
151
+ "dtype": "float32",
152
+ "format": "f32-to-bf16",
153
+ "nbytes": 1024,
154
+ "byteOffset": 6303744
155
+ },
156
+ {
157
+ "name": "gpt_neox.layers.1.input_layernorm.weight",
158
+ "shape": [
159
+ 512
160
+ ],
161
+ "dtype": "float16",
162
+ "format": "f32-to-bf16",
163
+ "nbytes": 1024,
164
+ "byteOffset": 6304768
165
+ },
166
+ {
167
+ "name": "gpt_neox.layers.1.input_layernorm.bias",
168
+ "shape": [
169
+ 512
170
+ ],
171
+ "dtype": "float16",
172
+ "format": "f32-to-bf16",
173
+ "nbytes": 1024,
174
+ "byteOffset": 6305792
175
+ },
176
+ {
177
+ "name": "gpt_neox.layers.1.post_attention_layernorm.weight",
178
+ "shape": [
179
+ 512
180
+ ],
181
+ "dtype": "float16",
182
+ "format": "f32-to-bf16",
183
+ "nbytes": 1024,
184
+ "byteOffset": 6306816
185
+ },
186
+ {
187
+ "name": "gpt_neox.layers.1.post_attention_layernorm.bias",
188
+ "shape": [
189
+ 512
190
+ ],
191
+ "dtype": "float16",
192
+ "format": "f32-to-bf16",
193
+ "nbytes": 1024,
194
+ "byteOffset": 6307840
195
+ },
196
+ {
197
+ "name": "gpt_neox.layers.1.attention.query_key_value.weight",
198
+ "shape": [
199
+ 1536,
200
+ 512
201
+ ],
202
+ "dtype": "float16",
203
+ "format": "f32-to-bf16",
204
+ "nbytes": 1572864,
205
+ "byteOffset": 6308864
206
+ },
207
+ {
208
+ "name": "gpt_neox.layers.1.attention.query_key_value.bias",
209
+ "shape": [
210
+ 1536
211
+ ],
212
+ "dtype": "float16",
213
+ "format": "f32-to-bf16",
214
+ "nbytes": 3072,
215
+ "byteOffset": 7881728
216
+ },
217
+ {
218
+ "name": "gpt_neox.layers.1.attention.dense.weight",
219
+ "shape": [
220
+ 512,
221
+ 512
222
+ ],
223
+ "dtype": "float16",
224
+ "format": "f32-to-bf16",
225
+ "nbytes": 524288,
226
+ "byteOffset": 7884800
227
+ },
228
+ {
229
+ "name": "gpt_neox.layers.1.attention.dense.bias",
230
+ "shape": [
231
+ 512
232
+ ],
233
+ "dtype": "float16",
234
+ "format": "f32-to-bf16",
235
+ "nbytes": 1024,
236
+ "byteOffset": 8409088
237
+ },
238
+ {
239
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.weight",
240
+ "shape": [
241
+ 2048,
242
+ 512
243
+ ],
244
+ "dtype": "float16",
245
+ "format": "f32-to-bf16",
246
+ "nbytes": 2097152,
247
+ "byteOffset": 8410112
248
+ },
249
+ {
250
+ "name": "gpt_neox.layers.1.mlp.dense_h_to_4h.bias",
251
+ "shape": [
252
+ 2048
253
+ ],
254
+ "dtype": "float32",
255
+ "format": "f32-to-bf16",
256
+ "nbytes": 4096,
257
+ "byteOffset": 10507264
258
+ },
259
+ {
260
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.weight",
261
+ "shape": [
262
+ 512,
263
+ 2048
264
+ ],
265
+ "dtype": "float16",
266
+ "format": "f32-to-bf16",
267
+ "nbytes": 2097152,
268
+ "byteOffset": 10511360
269
+ },
270
+ {
271
+ "name": "gpt_neox.layers.1.mlp.dense_4h_to_h.bias",
272
+ "shape": [
273
+ 512
274
+ ],
275
+ "dtype": "float32",
276
+ "format": "f32-to-bf16",
277
+ "nbytes": 1024,
278
+ "byteOffset": 12608512
279
+ },
280
+ {
281
+ "name": "gpt_neox.layers.2.input_layernorm.weight",
282
+ "shape": [
283
+ 512
284
+ ],
285
+ "dtype": "float16",
286
+ "format": "f32-to-bf16",
287
+ "nbytes": 1024,
288
+ "byteOffset": 12609536
289
+ },
290
+ {
291
+ "name": "gpt_neox.layers.2.input_layernorm.bias",
292
+ "shape": [
293
+ 512
294
+ ],
295
+ "dtype": "float16",
296
+ "format": "f32-to-bf16",
297
+ "nbytes": 1024,
298
+ "byteOffset": 12610560
299
+ },
300
+ {
301
+ "name": "gpt_neox.layers.2.post_attention_layernorm.weight",
302
+ "shape": [
303
+ 512
304
+ ],
305
+ "dtype": "float16",
306
+ "format": "f32-to-bf16",
307
+ "nbytes": 1024,
308
+ "byteOffset": 12611584
309
+ },
310
+ {
311
+ "name": "gpt_neox.layers.2.post_attention_layernorm.bias",
312
+ "shape": [
313
+ 512
314
+ ],
315
+ "dtype": "float16",
316
+ "format": "f32-to-bf16",
317
+ "nbytes": 1024,
318
+ "byteOffset": 12612608
319
+ },
320
+ {
321
+ "name": "gpt_neox.layers.2.attention.query_key_value.weight",
322
+ "shape": [
323
+ 1536,
324
+ 512
325
+ ],
326
+ "dtype": "float16",
327
+ "format": "f32-to-bf16",
328
+ "nbytes": 1572864,
329
+ "byteOffset": 12613632
330
+ },
331
+ {
332
+ "name": "gpt_neox.layers.2.attention.query_key_value.bias",
333
+ "shape": [
334
+ 1536
335
+ ],
336
+ "dtype": "float16",
337
+ "format": "f32-to-bf16",
338
+ "nbytes": 3072,
339
+ "byteOffset": 14186496
340
+ },
341
+ {
342
+ "name": "gpt_neox.layers.2.attention.dense.weight",
343
+ "shape": [
344
+ 512,
345
+ 512
346
+ ],
347
+ "dtype": "float16",
348
+ "format": "f32-to-bf16",
349
+ "nbytes": 524288,
350
+ "byteOffset": 14189568
351
+ },
352
+ {
353
+ "name": "gpt_neox.layers.2.attention.dense.bias",
354
+ "shape": [
355
+ 512
356
+ ],
357
+ "dtype": "float16",
358
+ "format": "f32-to-bf16",
359
+ "nbytes": 1024,
360
+ "byteOffset": 14713856
361
+ },
362
+ {
363
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.weight",
364
+ "shape": [
365
+ 2048,
366
+ 512
367
+ ],
368
+ "dtype": "float16",
369
+ "format": "f32-to-bf16",
370
+ "nbytes": 2097152,
371
+ "byteOffset": 14714880
372
+ },
373
+ {
374
+ "name": "gpt_neox.layers.2.mlp.dense_h_to_4h.bias",
375
+ "shape": [
376
+ 2048
377
+ ],
378
+ "dtype": "float32",
379
+ "format": "f32-to-bf16",
380
+ "nbytes": 4096,
381
+ "byteOffset": 16812032
382
+ },
383
+ {
384
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.weight",
385
+ "shape": [
386
+ 512,
387
+ 2048
388
+ ],
389
+ "dtype": "float16",
390
+ "format": "f32-to-bf16",
391
+ "nbytes": 2097152,
392
+ "byteOffset": 16816128
393
+ },
394
+ {
395
+ "name": "gpt_neox.layers.2.mlp.dense_4h_to_h.bias",
396
+ "shape": [
397
+ 512
398
+ ],
399
+ "dtype": "float32",
400
+ "format": "f32-to-bf16",
401
+ "nbytes": 1024,
402
+ "byteOffset": 18913280
403
+ },
404
+ {
405
+ "name": "gpt_neox.layers.3.input_layernorm.weight",
406
+ "shape": [
407
+ 512
408
+ ],
409
+ "dtype": "float16",
410
+ "format": "f32-to-bf16",
411
+ "nbytes": 1024,
412
+ "byteOffset": 18914304
413
+ },
414
+ {
415
+ "name": "gpt_neox.layers.3.input_layernorm.bias",
416
+ "shape": [
417
+ 512
418
+ ],
419
+ "dtype": "float16",
420
+ "format": "f32-to-bf16",
421
+ "nbytes": 1024,
422
+ "byteOffset": 18915328
423
+ },
424
+ {
425
+ "name": "gpt_neox.layers.3.post_attention_layernorm.weight",
426
+ "shape": [
427
+ 512
428
+ ],
429
+ "dtype": "float16",
430
+ "format": "f32-to-bf16",
431
+ "nbytes": 1024,
432
+ "byteOffset": 18916352
433
+ },
434
+ {
435
+ "name": "gpt_neox.layers.3.post_attention_layernorm.bias",
436
+ "shape": [
437
+ 512
438
+ ],
439
+ "dtype": "float16",
440
+ "format": "f32-to-bf16",
441
+ "nbytes": 1024,
442
+ "byteOffset": 18917376
443
+ },
444
+ {
445
+ "name": "gpt_neox.layers.3.attention.query_key_value.weight",
446
+ "shape": [
447
+ 1536,
448
+ 512
449
+ ],
450
+ "dtype": "float16",
451
+ "format": "f32-to-bf16",
452
+ "nbytes": 1572864,
453
+ "byteOffset": 18918400
454
+ },
455
+ {
456
+ "name": "gpt_neox.layers.3.attention.query_key_value.bias",
457
+ "shape": [
458
+ 1536
459
+ ],
460
+ "dtype": "float16",
461
+ "format": "f32-to-bf16",
462
+ "nbytes": 3072,
463
+ "byteOffset": 20491264
464
+ },
465
+ {
466
+ "name": "gpt_neox.layers.3.attention.dense.weight",
467
+ "shape": [
468
+ 512,
469
+ 512
470
+ ],
471
+ "dtype": "float16",
472
+ "format": "f32-to-bf16",
473
+ "nbytes": 524288,
474
+ "byteOffset": 20494336
475
+ },
476
+ {
477
+ "name": "gpt_neox.layers.3.attention.dense.bias",
478
+ "shape": [
479
+ 512
480
+ ],
481
+ "dtype": "float16",
482
+ "format": "f32-to-bf16",
483
+ "nbytes": 1024,
484
+ "byteOffset": 21018624
485
+ },
486
+ {
487
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.weight",
488
+ "shape": [
489
+ 2048,
490
+ 512
491
+ ],
492
+ "dtype": "float16",
493
+ "format": "f32-to-bf16",
494
+ "nbytes": 2097152,
495
+ "byteOffset": 21019648
496
+ },
497
+ {
498
+ "name": "gpt_neox.layers.3.mlp.dense_h_to_4h.bias",
499
+ "shape": [
500
+ 2048
501
+ ],
502
+ "dtype": "float32",
503
+ "format": "f32-to-bf16",
504
+ "nbytes": 4096,
505
+ "byteOffset": 23116800
506
+ },
507
+ {
508
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.weight",
509
+ "shape": [
510
+ 512,
511
+ 2048
512
+ ],
513
+ "dtype": "float16",
514
+ "format": "f32-to-bf16",
515
+ "nbytes": 2097152,
516
+ "byteOffset": 23120896
517
+ },
518
+ {
519
+ "name": "gpt_neox.layers.3.mlp.dense_4h_to_h.bias",
520
+ "shape": [
521
+ 512
522
+ ],
523
+ "dtype": "float32",
524
+ "format": "f32-to-bf16",
525
+ "nbytes": 1024,
526
+ "byteOffset": 25218048
527
+ },
528
+ {
529
+ "name": "gpt_neox.layers.4.input_layernorm.weight",
530
+ "shape": [
531
+ 512
532
+ ],
533
+ "dtype": "float16",
534
+ "format": "f32-to-bf16",
535
+ "nbytes": 1024,
536
+ "byteOffset": 25219072
537
+ },
538
+ {
539
+ "name": "gpt_neox.layers.4.input_layernorm.bias",
540
+ "shape": [
541
+ 512
542
+ ],
543
+ "dtype": "float16",
544
+ "format": "f32-to-bf16",
545
+ "nbytes": 1024,
546
+ "byteOffset": 25220096
547
+ },
548
+ {
549
+ "name": "gpt_neox.layers.4.post_attention_layernorm.weight",
550
+ "shape": [
551
+ 512
552
+ ],
553
+ "dtype": "float16",
554
+ "format": "f32-to-bf16",
555
+ "nbytes": 1024,
556
+ "byteOffset": 25221120
557
+ },
558
+ {
559
+ "name": "gpt_neox.layers.4.post_attention_layernorm.bias",
560
+ "shape": [
561
+ 512
562
+ ],
563
+ "dtype": "float16",
564
+ "format": "f32-to-bf16",
565
+ "nbytes": 1024,
566
+ "byteOffset": 25222144
567
+ },
568
+ {
569
+ "name": "gpt_neox.layers.4.attention.query_key_value.weight",
570
+ "shape": [
571
+ 1536,
572
+ 512
573
+ ],
574
+ "dtype": "float16",
575
+ "format": "f32-to-bf16",
576
+ "nbytes": 1572864,
577
+ "byteOffset": 25223168
578
+ },
579
+ {
580
+ "name": "gpt_neox.layers.4.attention.query_key_value.bias",
581
+ "shape": [
582
+ 1536
583
+ ],
584
+ "dtype": "float16",
585
+ "format": "f32-to-bf16",
586
+ "nbytes": 3072,
587
+ "byteOffset": 26796032
588
+ },
589
+ {
590
+ "name": "gpt_neox.layers.4.attention.dense.weight",
591
+ "shape": [
592
+ 512,
593
+ 512
594
+ ],
595
+ "dtype": "float16",
596
+ "format": "f32-to-bf16",
597
+ "nbytes": 524288,
598
+ "byteOffset": 26799104
599
+ },
600
+ {
601
+ "name": "gpt_neox.layers.4.attention.dense.bias",
602
+ "shape": [
603
+ 512
604
+ ],
605
+ "dtype": "float16",
606
+ "format": "f32-to-bf16",
607
+ "nbytes": 1024,
608
+ "byteOffset": 27323392
609
+ },
610
+ {
611
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.weight",
612
+ "shape": [
613
+ 2048,
614
+ 512
615
+ ],
616
+ "dtype": "float16",
617
+ "format": "f32-to-bf16",
618
+ "nbytes": 2097152,
619
+ "byteOffset": 27324416
620
+ },
621
+ {
622
+ "name": "gpt_neox.layers.4.mlp.dense_h_to_4h.bias",
623
+ "shape": [
624
+ 2048
625
+ ],
626
+ "dtype": "float32",
627
+ "format": "f32-to-bf16",
628
+ "nbytes": 4096,
629
+ "byteOffset": 29421568
630
+ },
631
+ {
632
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.weight",
633
+ "shape": [
634
+ 512,
635
+ 2048
636
+ ],
637
+ "dtype": "float16",
638
+ "format": "f32-to-bf16",
639
+ "nbytes": 2097152,
640
+ "byteOffset": 29425664
641
+ },
642
+ {
643
+ "name": "gpt_neox.layers.4.mlp.dense_4h_to_h.bias",
644
+ "shape": [
645
+ 512
646
+ ],
647
+ "dtype": "float32",
648
+ "format": "f32-to-bf16",
649
+ "nbytes": 1024,
650
+ "byteOffset": 31522816
651
+ },
652
+ {
653
+ "name": "gpt_neox.layers.5.input_layernorm.weight",
654
+ "shape": [
655
+ 512
656
+ ],
657
+ "dtype": "float16",
658
+ "format": "f32-to-bf16",
659
+ "nbytes": 1024,
660
+ "byteOffset": 31523840
661
+ },
662
+ {
663
+ "name": "gpt_neox.layers.5.input_layernorm.bias",
664
+ "shape": [
665
+ 512
666
+ ],
667
+ "dtype": "float16",
668
+ "format": "f32-to-bf16",
669
+ "nbytes": 1024,
670
+ "byteOffset": 31524864
671
+ },
672
+ {
673
+ "name": "gpt_neox.layers.5.post_attention_layernorm.weight",
674
+ "shape": [
675
+ 512
676
+ ],
677
+ "dtype": "float16",
678
+ "format": "f32-to-bf16",
679
+ "nbytes": 1024,
680
+ "byteOffset": 31525888
681
+ },
682
+ {
683
+ "name": "gpt_neox.layers.5.post_attention_layernorm.bias",
684
+ "shape": [
685
+ 512
686
+ ],
687
+ "dtype": "float16",
688
+ "format": "f32-to-bf16",
689
+ "nbytes": 1024,
690
+ "byteOffset": 31526912
691
+ },
692
+ {
693
+ "name": "gpt_neox.layers.5.attention.query_key_value.weight",
694
+ "shape": [
695
+ 1536,
696
+ 512
697
+ ],
698
+ "dtype": "float16",
699
+ "format": "f32-to-bf16",
700
+ "nbytes": 1572864,
701
+ "byteOffset": 31527936
702
+ },
703
+ {
704
+ "name": "gpt_neox.layers.5.attention.query_key_value.bias",
705
+ "shape": [
706
+ 1536
707
+ ],
708
+ "dtype": "float16",
709
+ "format": "f32-to-bf16",
710
+ "nbytes": 3072,
711
+ "byteOffset": 33100800
712
+ }
713
+ ],
714
+ "md5sum": "ef146e3d692ff0f92b4887051f5ce1aa"
715
+ },
716
+ {
717
+ "dataPath": "params_shard_2.bin",
718
+ "format": "raw-shard",
719
+ "nbytes": 51511296,
720
+ "records": [
721
+ {
722
+ "name": "embed_out.weight",
723
+ "shape": [
724
+ 50304,
725
+ 512
726
+ ],
727
+ "dtype": "float16",
728
+ "format": "f32-to-bf16",
729
+ "nbytes": 51511296,
730
+ "byteOffset": 0
731
+ }
732
+ ],
733
+ "md5sum": "2f016b975538d1e27f57fc84596c2432"
734
+ },
735
+ {
736
+ "dataPath": "params_shard_3.bin",
737
+ "format": "raw-shard",
738
+ "nbytes": 4726784,
739
+ "records": [
740
+ {
741
+ "name": "gpt_neox.layers.5.attention.dense.weight",
742
+ "shape": [
743
+ 512,
744
+ 512
745
+ ],
746
+ "dtype": "float16",
747
+ "format": "f32-to-bf16",
748
+ "nbytes": 524288,
749
+ "byteOffset": 0
750
+ },
751
+ {
752
+ "name": "gpt_neox.layers.5.attention.dense.bias",
753
+ "shape": [
754
+ 512
755
+ ],
756
+ "dtype": "float16",
757
+ "format": "f32-to-bf16",
758
+ "nbytes": 1024,
759
+ "byteOffset": 524288
760
+ },
761
+ {
762
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.weight",
763
+ "shape": [
764
+ 2048,
765
+ 512
766
+ ],
767
+ "dtype": "float16",
768
+ "format": "f32-to-bf16",
769
+ "nbytes": 2097152,
770
+ "byteOffset": 525312
771
+ },
772
+ {
773
+ "name": "gpt_neox.layers.5.mlp.dense_h_to_4h.bias",
774
+ "shape": [
775
+ 2048
776
+ ],
777
+ "dtype": "float32",
778
+ "format": "f32-to-bf16",
779
+ "nbytes": 4096,
780
+ "byteOffset": 2622464
781
+ },
782
+ {
783
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.weight",
784
+ "shape": [
785
+ 512,
786
+ 2048
787
+ ],
788
+ "dtype": "float16",
789
+ "format": "f32-to-bf16",
790
+ "nbytes": 2097152,
791
+ "byteOffset": 2626560
792
+ },
793
+ {
794
+ "name": "gpt_neox.layers.5.mlp.dense_4h_to_h.bias",
795
+ "shape": [
796
+ 512
797
+ ],
798
+ "dtype": "float32",
799
+ "format": "f32-to-bf16",
800
+ "nbytes": 1024,
801
+ "byteOffset": 4723712
802
+ },
803
+ {
804
+ "name": "gpt_neox.final_layer_norm.weight",
805
+ "shape": [
806
+ 512
807
+ ],
808
+ "dtype": "float16",
809
+ "format": "f32-to-bf16",
810
+ "nbytes": 1024,
811
+ "byteOffset": 4724736
812
+ },
813
+ {
814
+ "name": "gpt_neox.final_layer_norm.bias",
815
+ "shape": [
816
+ 512
817
+ ],
818
+ "dtype": "float16",
819
+ "format": "f32-to-bf16",
820
+ "nbytes": 1024,
821
+ "byteOffset": 4725760
822
+ }
823
+ ],
824
+ "md5sum": "4b204eca2568a39d4f462b042c0e9ab7"
825
+ }
826
+ ]
827
+ }
params_shard_0.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ffd61a6690646d9a8c5e43272a2b10de676169d994f7fc751baa9f3f45ec9faa
3
+ size 51511296
params_shard_1.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4ed2d742eb9c82c06f75581ece2f857cc02204c61930258c27a1ebe435966a2b
3
+ size 33103872
params_shard_2.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fca6e0e12182d051cdea7ecebc9553fc5c902fb5dac3b2bdb5f8f67fa6792a0f
3
+ size 51511296
params_shard_3.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d765b0b5abf3c230cbce12d35baba9eea458830a468edb73c86a97cd442b5b1
3
+ size 4726784
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "bos_token": "<|endoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "name_or_path": "EleutherAI/gpt-neox-20b",
6
+ "special_tokens_map_file": "/admin/home-hailey/.cache/huggingface/hub/models--EleutherAI--gpt-neox-20b/snapshots/4e49eadb5d14bd22f314ec3f45b69a87b88c7691/special_tokens_map.json",
7
+ "tokenizer_class": "GPTNeoXTokenizer",
8
+ "unk_token": "<|endoftext|>"
9
+ }