alexwengg commited on
Commit
1be4eb9
·
verified ·
1 Parent(s): 8418896

Upload reports/verification-multilingual-L1024.json with huggingface_hub

Browse files
reports/verification-multilingual-L1024.json ADDED
@@ -0,0 +1,579 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_fp16_L1024_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "88a745e53bdb53abdf600fa235fd7930ee8f9d59e7f1485f7402f036aaa17d77",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "a99aadbb5f3e3a6ad0ebf28df2027fdc9fae2717e2c7a2fb4c1e9ec8be1e9529",
6
+ "Manifest.json": "a4cb1a599ac7ee392f09181ff0f340df4a05540b9ff1566f15ee35646fb11e43"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 1024,
12
+ "max_options": 32,
13
+ "environment": {
14
+ "chip": "Apple M5 Pro",
15
+ "os": "27.0",
16
+ "python": "3.12.13",
17
+ "torch": "2.7.0",
18
+ "coremltools": "9.0"
19
+ },
20
+ "gates": {
21
+ "argmax_agreement": "100%",
22
+ "max_probability_error": 0.02,
23
+ "max_action_probability_error": 0.02,
24
+ "finite": true,
25
+ "padding_masked": true
26
+ },
27
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
28
+ "runs": {
29
+ "ALL": {
30
+ "compute_units": "ALL",
31
+ "load_seconds": 1.9747608749894425,
32
+ "questions": 16,
33
+ "argmax_agreements": 16,
34
+ "max_probability_error": 0.002333313226699829,
35
+ "max_action_probability_error": 0.0,
36
+ "latency_ms": {
37
+ "p50": 17.886937508592382,
38
+ "p95": 18.32043749454897,
39
+ "min": 17.64466700842604,
40
+ "max": 23.31449999473989
41
+ },
42
+ "skipped": [],
43
+ "rows": [
44
+ {
45
+ "case": "tetris_clean",
46
+ "question": "clean",
47
+ "type": "noul",
48
+ "tokens": 80,
49
+ "options": 2,
50
+ "argmax_agrees": true,
51
+ "coreml_argmax": 0,
52
+ "reference_argmax": 0,
53
+ "max_probability_error": 8.064508438110352e-05,
54
+ "max_action_probability_error": 0.0,
55
+ "finite": true,
56
+ "padding_masked": true,
57
+ "probability_sum": 0.9998779296875,
58
+ "median_ms": 17.863854009192437
59
+ },
60
+ {
61
+ "case": "tetris_flat",
62
+ "question": "clean",
63
+ "type": "noul",
64
+ "tokens": 84,
65
+ "options": 2,
66
+ "argmax_agrees": true,
67
+ "coreml_argmax": 1,
68
+ "reference_argmax": 1,
69
+ "max_probability_error": 0.00014770030975341797,
70
+ "max_action_probability_error": 0.0,
71
+ "finite": true,
72
+ "padding_masked": true,
73
+ "probability_sum": 0.9999237060546875,
74
+ "median_ms": 18.013292021350935
75
+ },
76
+ {
77
+ "case": "tetris_lane",
78
+ "question": "lane",
79
+ "type": "choice",
80
+ "tokens": 62,
81
+ "options": 4,
82
+ "argmax_agrees": true,
83
+ "coreml_argmax": 1,
84
+ "reference_argmax": 1,
85
+ "max_probability_error": 0.0007520020008087158,
86
+ "max_action_probability_error": 0.0,
87
+ "finite": true,
88
+ "padding_masked": true,
89
+ "probability_sum": 1.0,
90
+ "median_ms": 17.79310347046703
91
+ },
92
+ {
93
+ "case": "tetris_lane",
94
+ "question": "risk",
95
+ "type": "score",
96
+ "tokens": 64,
97
+ "options": 3,
98
+ "argmax_agrees": true,
99
+ "coreml_argmax": 1,
100
+ "reference_argmax": 1,
101
+ "max_probability_error": 0.00046347081661224365,
102
+ "max_action_probability_error": 0.0,
103
+ "finite": true,
104
+ "padding_masked": true,
105
+ "probability_sum": 1.0,
106
+ "median_ms": 17.958333482965827
107
+ },
108
+ {
109
+ "case": "form_field",
110
+ "question": "action",
111
+ "type": "choice",
112
+ "tokens": 104,
113
+ "options": 3,
114
+ "argmax_agrees": true,
115
+ "coreml_argmax": 0,
116
+ "reference_argmax": 0,
117
+ "max_probability_error": 0.00038102269172668457,
118
+ "max_action_probability_error": 0.0,
119
+ "finite": true,
120
+ "padding_masked": true,
121
+ "probability_sum": 0.9998779296875,
122
+ "median_ms": 17.863124987343326
123
+ },
124
+ {
125
+ "case": "support_triage",
126
+ "question": "intent",
127
+ "type": "choice",
128
+ "tokens": 62,
129
+ "options": 4,
130
+ "argmax_agrees": true,
131
+ "coreml_argmax": 1,
132
+ "reference_argmax": 1,
133
+ "max_probability_error": 5.608797073364258e-05,
134
+ "max_action_probability_error": 0.0,
135
+ "finite": true,
136
+ "padding_masked": true,
137
+ "probability_sum": 1.0000487565994263,
138
+ "median_ms": 17.887624504510313
139
+ },
140
+ {
141
+ "case": "support_triage",
142
+ "question": "frustration",
143
+ "type": "score",
144
+ "tokens": 69,
145
+ "options": 3,
146
+ "argmax_agrees": true,
147
+ "coreml_argmax": 1,
148
+ "reference_argmax": 1,
149
+ "max_probability_error": 0.0005022585391998291,
150
+ "max_action_probability_error": 0.0,
151
+ "finite": true,
152
+ "padding_masked": true,
153
+ "probability_sum": 1.0,
154
+ "median_ms": 17.897104495204985
155
+ },
156
+ {
157
+ "case": "support_triage",
158
+ "question": "churn",
159
+ "type": "noul",
160
+ "tokens": 71,
161
+ "options": 2,
162
+ "argmax_agrees": true,
163
+ "coreml_argmax": 0,
164
+ "reference_argmax": 0,
165
+ "max_probability_error": 3.820657730102539e-05,
166
+ "max_action_probability_error": 0.0,
167
+ "finite": true,
168
+ "padding_masked": true,
169
+ "probability_sum": 0.9999771118164062,
170
+ "median_ms": 17.81887552351691
171
+ },
172
+ {
173
+ "case": "zh",
174
+ "question": "intent",
175
+ "type": "choice",
176
+ "tokens": 37,
177
+ "options": 3,
178
+ "argmax_agrees": true,
179
+ "coreml_argmax": 0,
180
+ "reference_argmax": 0,
181
+ "max_probability_error": 0.00023287534713745117,
182
+ "max_action_probability_error": 0.0,
183
+ "finite": true,
184
+ "padding_masked": true,
185
+ "probability_sum": 0.999762773513794,
186
+ "median_ms": 17.968854488572106
187
+ },
188
+ {
189
+ "case": "zh",
190
+ "question": "urgent",
191
+ "type": "noul",
192
+ "tokens": 48,
193
+ "options": 2,
194
+ "argmax_agrees": true,
195
+ "coreml_argmax": 1,
196
+ "reference_argmax": 1,
197
+ "max_probability_error": 0.00012467056512832642,
198
+ "max_action_probability_error": 0.0,
199
+ "finite": true,
200
+ "padding_masked": true,
201
+ "probability_sum": 0.999755859375,
202
+ "median_ms": 17.823145462898538
203
+ },
204
+ {
205
+ "case": "ja",
206
+ "question": "intent",
207
+ "type": "choice",
208
+ "tokens": 45,
209
+ "options": 3,
210
+ "argmax_agrees": true,
211
+ "coreml_argmax": 0,
212
+ "reference_argmax": 0,
213
+ "max_probability_error": 0.00017899274826049805,
214
+ "max_action_probability_error": 0.0,
215
+ "finite": true,
216
+ "padding_masked": true,
217
+ "probability_sum": 1.000178337097168,
218
+ "median_ms": 17.794958519516513
219
+ },
220
+ {
221
+ "case": "de",
222
+ "question": "intent",
223
+ "type": "choice",
224
+ "tokens": 44,
225
+ "options": 3,
226
+ "argmax_agrees": true,
227
+ "coreml_argmax": 0,
228
+ "reference_argmax": 0,
229
+ "max_probability_error": 7.051229476928711e-05,
230
+ "max_action_probability_error": 0.0,
231
+ "finite": true,
232
+ "padding_masked": true,
233
+ "probability_sum": 0.9999675750732422,
234
+ "median_ms": 17.900292004924268
235
+ },
236
+ {
237
+ "case": "many_options",
238
+ "question": "lane",
239
+ "type": "choice",
240
+ "tokens": 157,
241
+ "options": 20,
242
+ "argmax_agrees": true,
243
+ "coreml_argmax": 3,
244
+ "reference_argmax": 3,
245
+ "max_probability_error": 0.000586189329624176,
246
+ "max_action_probability_error": 0.0,
247
+ "finite": true,
248
+ "padding_masked": true,
249
+ "probability_sum": 0.9999237060546875,
250
+ "median_ms": 17.867436981759965
251
+ },
252
+ {
253
+ "case": "guard",
254
+ "question": "injection",
255
+ "type": "noul",
256
+ "tokens": 48,
257
+ "options": 2,
258
+ "argmax_agrees": true,
259
+ "coreml_argmax": 1,
260
+ "reference_argmax": 1,
261
+ "max_probability_error": 0.002333313226699829,
262
+ "max_action_probability_error": 0.0,
263
+ "finite": true,
264
+ "padding_masked": true,
265
+ "probability_sum": 1.000244140625,
266
+ "median_ms": 17.970292014069855
267
+ },
268
+ {
269
+ "case": "long_state",
270
+ "question": "ready",
271
+ "type": "noul",
272
+ "tokens": 174,
273
+ "options": 2,
274
+ "argmax_agrees": true,
275
+ "coreml_argmax": 1,
276
+ "reference_argmax": 1,
277
+ "max_probability_error": 0.0001442432403564453,
278
+ "max_action_probability_error": 0.0,
279
+ "finite": true,
280
+ "padding_masked": true,
281
+ "probability_sum": 1.00006103515625,
282
+ "median_ms": 17.907041998114437
283
+ },
284
+ {
285
+ "case": "long_state",
286
+ "question": "owner",
287
+ "type": "choice",
288
+ "tokens": 158,
289
+ "options": 3,
290
+ "argmax_agrees": true,
291
+ "coreml_argmax": 2,
292
+ "reference_argmax": 2,
293
+ "max_probability_error": 0.00044977664947509766,
294
+ "max_action_probability_error": 0.0,
295
+ "finite": true,
296
+ "padding_masked": true,
297
+ "probability_sum": 1.00018310546875,
298
+ "median_ms": 17.80535452417098
299
+ }
300
+ ],
301
+ "passed": true
302
+ },
303
+ "CPU_AND_NE": {
304
+ "compute_units": "CPU_AND_NE",
305
+ "load_seconds": 6.936569582961965,
306
+ "questions": 16,
307
+ "argmax_agreements": 16,
308
+ "max_probability_error": 0.012559235095977783,
309
+ "max_action_probability_error": 0.0,
310
+ "latency_ms": {
311
+ "p50": 80.06945901433937,
312
+ "p95": 80.43368989310693,
313
+ "min": 79.77154199033976,
314
+ "max": 80.92245802981779
315
+ },
316
+ "skipped": [],
317
+ "rows": [
318
+ {
319
+ "case": "tetris_clean",
320
+ "question": "clean",
321
+ "type": "noul",
322
+ "tokens": 80,
323
+ "options": 2,
324
+ "argmax_agrees": true,
325
+ "coreml_argmax": 0,
326
+ "reference_argmax": 0,
327
+ "max_probability_error": 0.0004076361656188965,
328
+ "max_action_probability_error": 0.0,
329
+ "finite": true,
330
+ "padding_masked": true,
331
+ "probability_sum": 1.00018310546875,
332
+ "median_ms": 80.10460447985679
333
+ },
334
+ {
335
+ "case": "tetris_flat",
336
+ "question": "clean",
337
+ "type": "noul",
338
+ "tokens": 84,
339
+ "options": 2,
340
+ "argmax_agrees": true,
341
+ "coreml_argmax": 1,
342
+ "reference_argmax": 1,
343
+ "max_probability_error": 0.00034058094024658203,
344
+ "max_action_probability_error": 0.0,
345
+ "finite": true,
346
+ "padding_masked": true,
347
+ "probability_sum": 1.0003662109375,
348
+ "median_ms": 80.07691701641306
349
+ },
350
+ {
351
+ "case": "tetris_lane",
352
+ "question": "lane",
353
+ "type": "choice",
354
+ "tokens": 62,
355
+ "options": 4,
356
+ "argmax_agrees": true,
357
+ "coreml_argmax": 1,
358
+ "reference_argmax": 1,
359
+ "max_probability_error": 0.0033596158027648926,
360
+ "max_action_probability_error": 0.0,
361
+ "finite": true,
362
+ "padding_masked": true,
363
+ "probability_sum": 0.9998779296875,
364
+ "median_ms": 80.06479148752987
365
+ },
366
+ {
367
+ "case": "tetris_lane",
368
+ "question": "risk",
369
+ "type": "score",
370
+ "tokens": 64,
371
+ "options": 3,
372
+ "argmax_agrees": true,
373
+ "coreml_argmax": 1,
374
+ "reference_argmax": 1,
375
+ "max_probability_error": 0.0014896541833877563,
376
+ "max_action_probability_error": 0.0,
377
+ "finite": true,
378
+ "padding_masked": true,
379
+ "probability_sum": 0.9996337890625,
380
+ "median_ms": 80.11591649847105
381
+ },
382
+ {
383
+ "case": "form_field",
384
+ "question": "action",
385
+ "type": "choice",
386
+ "tokens": 104,
387
+ "options": 3,
388
+ "argmax_agrees": true,
389
+ "coreml_argmax": 0,
390
+ "reference_argmax": 0,
391
+ "max_probability_error": 0.0034949183464050293,
392
+ "max_action_probability_error": 0.0,
393
+ "finite": true,
394
+ "padding_masked": true,
395
+ "probability_sum": 0.9996337890625,
396
+ "median_ms": 80.14208299573511
397
+ },
398
+ {
399
+ "case": "support_triage",
400
+ "question": "intent",
401
+ "type": "choice",
402
+ "tokens": 62,
403
+ "options": 4,
404
+ "argmax_agrees": true,
405
+ "coreml_argmax": 1,
406
+ "reference_argmax": 1,
407
+ "max_probability_error": 0.0004321932792663574,
408
+ "max_action_probability_error": 0.0,
409
+ "finite": true,
410
+ "padding_masked": true,
411
+ "probability_sum": 0.9995682239532471,
412
+ "median_ms": 80.23072901414707
413
+ },
414
+ {
415
+ "case": "support_triage",
416
+ "question": "frustration",
417
+ "type": "score",
418
+ "tokens": 69,
419
+ "options": 3,
420
+ "argmax_agrees": true,
421
+ "coreml_argmax": 1,
422
+ "reference_argmax": 1,
423
+ "max_probability_error": 0.001600891351699829,
424
+ "max_action_probability_error": 0.0,
425
+ "finite": true,
426
+ "padding_masked": true,
427
+ "probability_sum": 0.99981689453125,
428
+ "median_ms": 80.22379150497727
429
+ },
430
+ {
431
+ "case": "support_triage",
432
+ "question": "churn",
433
+ "type": "noul",
434
+ "tokens": 71,
435
+ "options": 2,
436
+ "argmax_agrees": true,
437
+ "coreml_argmax": 0,
438
+ "reference_argmax": 0,
439
+ "max_probability_error": 3.820657730102539e-05,
440
+ "max_action_probability_error": 0.0,
441
+ "finite": true,
442
+ "padding_masked": true,
443
+ "probability_sum": 0.9999771118164062,
444
+ "median_ms": 80.06289551849477
445
+ },
446
+ {
447
+ "case": "zh",
448
+ "question": "intent",
449
+ "type": "choice",
450
+ "tokens": 37,
451
+ "options": 3,
452
+ "argmax_agrees": true,
453
+ "coreml_argmax": 0,
454
+ "reference_argmax": 0,
455
+ "max_probability_error": 0.00023287534713745117,
456
+ "max_action_probability_error": 0.0,
457
+ "finite": true,
458
+ "padding_masked": true,
459
+ "probability_sum": 0.999773383140564,
460
+ "median_ms": 80.02085401676595
461
+ },
462
+ {
463
+ "case": "zh",
464
+ "question": "urgent",
465
+ "type": "noul",
466
+ "tokens": 48,
467
+ "options": 2,
468
+ "argmax_agrees": true,
469
+ "coreml_argmax": 1,
470
+ "reference_argmax": 1,
471
+ "max_probability_error": 0.0004093870520591736,
472
+ "max_action_probability_error": 0.0,
473
+ "finite": true,
474
+ "padding_masked": true,
475
+ "probability_sum": 1.0002899169921875,
476
+ "median_ms": 79.99004202429205
477
+ },
478
+ {
479
+ "case": "ja",
480
+ "question": "intent",
481
+ "type": "choice",
482
+ "tokens": 45,
483
+ "options": 3,
484
+ "argmax_agrees": true,
485
+ "coreml_argmax": 0,
486
+ "reference_argmax": 0,
487
+ "max_probability_error": 0.00017899274826049805,
488
+ "max_action_probability_error": 0.0,
489
+ "finite": true,
490
+ "padding_masked": true,
491
+ "probability_sum": 1.0001769065856934,
492
+ "median_ms": 80.35356248728931
493
+ },
494
+ {
495
+ "case": "de",
496
+ "question": "intent",
497
+ "type": "choice",
498
+ "tokens": 44,
499
+ "options": 3,
500
+ "argmax_agrees": true,
501
+ "coreml_argmax": 0,
502
+ "reference_argmax": 0,
503
+ "max_probability_error": 7.051229476928711e-05,
504
+ "max_action_probability_error": 0.0,
505
+ "finite": true,
506
+ "padding_masked": true,
507
+ "probability_sum": 0.9998779296875,
508
+ "median_ms": 80.08831250481308
509
+ },
510
+ {
511
+ "case": "many_options",
512
+ "question": "lane",
513
+ "type": "choice",
514
+ "tokens": 157,
515
+ "options": 20,
516
+ "argmax_agrees": true,
517
+ "coreml_argmax": 3,
518
+ "reference_argmax": 3,
519
+ "max_probability_error": 0.001290060579776764,
520
+ "max_action_probability_error": 0.0,
521
+ "finite": true,
522
+ "padding_masked": true,
523
+ "probability_sum": 0.9998397827148438,
524
+ "median_ms": 79.97343697934411
525
+ },
526
+ {
527
+ "case": "guard",
528
+ "question": "injection",
529
+ "type": "noul",
530
+ "tokens": 48,
531
+ "options": 2,
532
+ "argmax_agrees": true,
533
+ "coreml_argmax": 1,
534
+ "reference_argmax": 1,
535
+ "max_probability_error": 0.012559235095977783,
536
+ "max_action_probability_error": 0.0,
537
+ "finite": true,
538
+ "padding_masked": true,
539
+ "probability_sum": 1.000244140625,
540
+ "median_ms": 79.98441651579924
541
+ },
542
+ {
543
+ "case": "long_state",
544
+ "question": "ready",
545
+ "type": "noul",
546
+ "tokens": 174,
547
+ "options": 2,
548
+ "argmax_agrees": true,
549
+ "coreml_argmax": 1,
550
+ "reference_argmax": 1,
551
+ "max_probability_error": 0.0003440380096435547,
552
+ "max_action_probability_error": 0.0,
553
+ "finite": true,
554
+ "padding_masked": true,
555
+ "probability_sum": 0.999664306640625,
556
+ "median_ms": 80.0024795171339
557
+ },
558
+ {
559
+ "case": "long_state",
560
+ "question": "owner",
561
+ "type": "choice",
562
+ "tokens": 158,
563
+ "options": 3,
564
+ "argmax_agrees": true,
565
+ "coreml_argmax": 2,
566
+ "reference_argmax": 2,
567
+ "max_probability_error": 0.0005267858505249023,
568
+ "max_action_probability_error": 0.0,
569
+ "finite": true,
570
+ "padding_masked": true,
571
+ "probability_sum": 0.999908447265625,
572
+ "median_ms": 79.98839602805674
573
+ }
574
+ ],
575
+ "passed": true
576
+ }
577
+ },
578
+ "passed": true
579
+ }