File size: 18,268 Bytes
7ce899b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
{
  "package": "laya_multilingual_fp16_L128_options32.mlpackage",
  "package_files_sha256": {
    "Data/com.apple.CoreML/model.mlmodel": "a536e177f174cc14a0dbe539357db29016a10ad579cdbf040b98525f32e7a4a7",
    "Data/com.apple.CoreML/weights/weight.bin": "506eb53e74aaebcdfb3f55304fbdd0b9c1678026efaef60454a1c66557f35d90",
    "Manifest.json": "9957a84277c0a722bcdbbc69ae3ce065122d9dd14de85ee9c76a2e44b2cb7146"
  },
  "source_repo": "convaiinnovations/laya",
  "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
  "variant": "multilingual",
  "length": 128,
  "max_options": 32,
  "environment": {
    "chip": "Apple M5 Pro",
    "os": "27.0",
    "python": "3.12.13",
    "torch": "2.7.0",
    "coremltools": "9.0"
  },
  "gates": {
    "argmax_agreement": "100%",
    "max_probability_error": 0.02,
    "max_action_probability_error": 0.02,
    "finite": true,
    "padding_masked": true
  },
  "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
  "runs": {
    "ALL": {
      "compute_units": "ALL",
      "load_seconds": 1.9205929159652442,
      "questions": 16,
      "argmax_agreements": 16,
      "max_probability_error": 0.0021350979804992676,
      "max_action_probability_error": 0.0,
      "latency_ms": {
        "p50": 3.874978981912136,
        "p95": 4.150477421353571,
        "min": 3.734458005055785,
        "max": 6.0334589798003435
      },
      "skipped": [],
      "rows": [
        {
          "case": "tetris_clean",
          "question": "clean",
          "type": "noul",
          "tokens": 80,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0001417025923728943,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00006103515625,
          "median_ms": 3.948333003791049
        },
        {
          "case": "tetris_flat",
          "question": "clean",
          "type": "noul",
          "tokens": 84,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.00014770030975341797,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998626708984375,
          "median_ms": 3.872645494993776
        },
        {
          "case": "tetris_lane",
          "question": "lane",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0005466341972351074,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001220703125,
          "median_ms": 4.006020986707881
        },
        {
          "case": "tetris_lane",
          "question": "risk",
          "type": "score",
          "tokens": 64,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.00014688074588775635,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 3.9132919919211417
        },
        {
          "case": "form_field",
          "question": "action",
          "type": "choice",
          "tokens": 104,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0021350979804992676,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 3.8335000281222165
        },
        {
          "case": "support_triage",
          "question": "intent",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 5.608797073364258e-05,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0000602006912231,
          "median_ms": 3.799437516136095
        },
        {
          "case": "support_triage",
          "question": "frustration",
          "type": "score",
          "tokens": 69,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0008405148983001709,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00006103515625,
          "median_ms": 3.865458507789299
        },
        {
          "case": "support_triage",
          "question": "churn",
          "type": "noul",
          "tokens": 71,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 3.820657730102539e-05,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9999732971191406,
          "median_ms": 3.829437482636422
        },
        {
          "case": "zh",
          "question": "intent",
          "type": "choice",
          "tokens": 37,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00023287534713745117,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9997743368148804,
          "median_ms": 3.8122705300338566
        },
        {
          "case": "zh",
          "question": "urgent",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.00011944770812988281,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999786376953125,
          "median_ms": 3.9022920245770365
        },
        {
          "case": "ja",
          "question": "intent",
          "type": "choice",
          "tokens": 45,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00017899274826049805,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000178337097168,
          "median_ms": 3.817520511802286
        },
        {
          "case": "de",
          "question": "intent",
          "type": "choice",
          "tokens": 44,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 7.051229476928711e-05,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998912811279297,
          "median_ms": 3.8392704736907035
        },
        {
          "case": "many_options",
          "question": "lane",
          "type": "choice",
          "tokens": 128,
          "options": 20,
          "argmax_agrees": true,
          "coreml_argmax": 3,
          "reference_argmax": 3,
          "max_probability_error": 0.0005449950695037842,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9999656677246094,
          "median_ms": 4.023146029794589
        },
        {
          "case": "guard",
          "question": "injection",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0011126399040222168,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0,
          "median_ms": 3.868666506605223
        },
        {
          "case": "long_state",
          "question": "ready",
          "type": "noul",
          "tokens": 128,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0001234356313943863,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999847412109375,
          "median_ms": 3.8872919976711273
        },
        {
          "case": "long_state",
          "question": "owner",
          "type": "choice",
          "tokens": 128,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 2,
          "reference_argmax": 2,
          "max_probability_error": 0.0010861754417419434,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9999961853027344,
          "median_ms": 3.830937494058162
        }
      ],
      "passed": true
    },
    "CPU_AND_NE": {
      "compute_units": "CPU_AND_NE",
      "load_seconds": 3.0328945000073873,
      "questions": 16,
      "argmax_agreements": 16,
      "max_probability_error": 0.012559235095977783,
      "max_action_probability_error": 0.0,
      "latency_ms": {
        "p50": 3.6408329906407744,
        "p95": 3.7919832742772996,
        "min": 3.5940000088885427,
        "max": 6.553583953063935
      },
      "skipped": [],
      "rows": [
        {
          "case": "tetris_clean",
          "question": "clean",
          "type": "noul",
          "tokens": 80,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0004076361656188965,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00018310546875,
          "median_ms": 3.6329374997876585
        },
        {
          "case": "tetris_flat",
          "question": "clean",
          "type": "noul",
          "tokens": 84,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.00034058094024658203,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0003662109375,
          "median_ms": 3.6503754963632673
        },
        {
          "case": "tetris_lane",
          "question": "lane",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0033596158027648926,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 3.6310830037109554
        },
        {
          "case": "tetris_lane",
          "question": "risk",
          "type": "score",
          "tokens": 64,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0014896541833877563,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9996337890625,
          "median_ms": 3.644520998932421
        },
        {
          "case": "form_field",
          "question": "action",
          "type": "choice",
          "tokens": 104,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.0034949183464050293,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9996337890625,
          "median_ms": 3.6403754784259945
        },
        {
          "case": "support_triage",
          "question": "intent",
          "type": "choice",
          "tokens": 62,
          "options": 4,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0004321932792663574,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9995682239532471,
          "median_ms": 3.631958010373637
        },
        {
          "case": "support_triage",
          "question": "frustration",
          "type": "score",
          "tokens": 69,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.001600891351699829,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.99981689453125,
          "median_ms": 3.6398124939296395
        },
        {
          "case": "support_triage",
          "question": "churn",
          "type": "noul",
          "tokens": 71,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 3.820657730102539e-05,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9999771118164062,
          "median_ms": 3.6401874967850745
        },
        {
          "case": "zh",
          "question": "intent",
          "type": "choice",
          "tokens": 37,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00023287534713745117,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.999773383140564,
          "median_ms": 3.7934584834147245
        },
        {
          "case": "zh",
          "question": "urgent",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0004093870520591736,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0002899169921875,
          "median_ms": 3.753063007025048
        },
        {
          "case": "ja",
          "question": "intent",
          "type": "choice",
          "tokens": 45,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 0.00017899274826049805,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0001769065856934,
          "median_ms": 3.62402101745829
        },
        {
          "case": "de",
          "question": "intent",
          "type": "choice",
          "tokens": 44,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 0,
          "reference_argmax": 0,
          "max_probability_error": 7.051229476928711e-05,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 0.9998779296875,
          "median_ms": 3.6391039902810007
        },
        {
          "case": "many_options",
          "question": "lane",
          "type": "choice",
          "tokens": 128,
          "options": 20,
          "argmax_agrees": true,
          "coreml_argmax": 3,
          "reference_argmax": 3,
          "max_probability_error": 0.004200160503387451,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0005359649658203,
          "median_ms": 3.6420005199033767
        },
        {
          "case": "guard",
          "question": "injection",
          "type": "noul",
          "tokens": 48,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.012559235095977783,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.000244140625,
          "median_ms": 3.6424169957172126
        },
        {
          "case": "long_state",
          "question": "ready",
          "type": "noul",
          "tokens": 128,
          "options": 2,
          "argmax_agrees": true,
          "coreml_argmax": 1,
          "reference_argmax": 1,
          "max_probability_error": 0.0002122577279806137,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.00018310546875,
          "median_ms": 3.634604043327272
        },
        {
          "case": "long_state",
          "question": "owner",
          "type": "choice",
          "tokens": 128,
          "options": 3,
          "argmax_agrees": true,
          "coreml_argmax": 2,
          "reference_argmax": 2,
          "max_probability_error": 0.0022311657667160034,
          "max_action_probability_error": 0.0,
          "finite": true,
          "padding_masked": true,
          "probability_sum": 1.0003395080566406,
          "median_ms": 3.628145990660414
        }
      ],
      "passed": true
    }
  },
  "passed": true
}