alexwengg commited on
Commit
b7d5f10
·
verified ·
1 Parent(s): b4c3487

Add e8 + compression + ANE profile reports

Browse files
reports/ane-profile-L1024.json ADDED
The diff for this file is too large to render. See raw diff
 
reports/ane-profile-L128.json ADDED
The diff for this file is too large to render. See raw diff
 
reports/ane-profile-L256.json ADDED
The diff for this file is too large to render. See raw diff
 
reports/ane-profile-L512.json ADDED
The diff for this file is too large to render. See raw diff
 
reports/benchmark-coreml-e8.json ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "chip" : "Apple M5 Pro",
3
+ "elapsed_s" : 127.73304998874664,
4
+ "latency_ms" : {
5
+ "p50" : 5.3043329999999997,
6
+ "p95" : 18.064624999999999
7
+ },
8
+ "lengths" : [
9
+ 128,
10
+ 256,
11
+ 512,
12
+ 1024
13
+ ],
14
+ "load_s" : 30.526584982872009,
15
+ "questions" : 3899,
16
+ "suites" : {
17
+ "app.email_spam" : {
18
+ "accuracy" : 0.99250000000000005,
19
+ "buckets" : {
20
+ "L128" : 98,
21
+ "L256" : 117,
22
+ "L512" : 93,
23
+ "L1024" : 92
24
+ },
25
+ "dropped" : 0,
26
+ "latency_ms" : {
27
+ "p50" : 7.062875,
28
+ "p95" : 18.560207999999999
29
+ },
30
+ "max_probability_delta_vs_reference" : 0.024192176759243011,
31
+ "n" : 400,
32
+ "reference_argmax_agreement" : 1,
33
+ "reference_compared" : 400,
34
+ "state_truncated" : 9
35
+ },
36
+ "app.guardrails_jailbreak" : {
37
+ "accuracy" : 0.81000000000000005,
38
+ "buckets" : {
39
+ "L128" : 285,
40
+ "L256" : 59,
41
+ "L512" : 53,
42
+ "L1024" : 3
43
+ },
44
+ "dropped" : 0,
45
+ "latency_ms" : {
46
+ "p50" : 4.0982079999999996,
47
+ "p95" : 9.5879580000000004
48
+ },
49
+ "max_probability_delta_vs_reference" : 0.80097097158432007,
50
+ "n" : 400,
51
+ "reference_argmax_agreement" : 0.98999999999999999,
52
+ "reference_compared" : 400,
53
+ "state_truncated" : 0
54
+ },
55
+ "app.model_routing_domain" : {
56
+ "accuracy" : 0.45363408521303256,
57
+ "buckets" : {
58
+ "L128" : 137,
59
+ "L256" : 261,
60
+ "L512" : 1
61
+ },
62
+ "dropped" : 0,
63
+ "latency_ms" : {
64
+ "p50" : 5.6524590000000003,
65
+ "p95" : 6.3639999999999999
66
+ },
67
+ "max_probability_delta_vs_reference" : 0.077826619148254395,
68
+ "n" : 399,
69
+ "reference_argmax_agreement" : 0.97493734335839599,
70
+ "reference_compared" : 399,
71
+ "state_truncated" : 0
72
+ },
73
+ "app.moderation_toxicity" : {
74
+ "accuracy" : 0.53500000000000003,
75
+ "buckets" : {
76
+ "L128" : 306,
77
+ "L256" : 54,
78
+ "L512" : 37,
79
+ "L1024" : 3
80
+ },
81
+ "dropped" : 0,
82
+ "latency_ms" : {
83
+ "p50" : 4.0458749999999997,
84
+ "p95" : 10.238166
85
+ },
86
+ "max_probability_delta_vs_reference" : 0.016416072845458984,
87
+ "n" : 400,
88
+ "reference_argmax_agreement" : 1,
89
+ "reference_compared" : 400,
90
+ "state_truncated" : 0
91
+ },
92
+ "app.phishing" : {
93
+ "accuracy" : 0.99250000000000005,
94
+ "buckets" : {
95
+ "L128" : 76,
96
+ "L256" : 111,
97
+ "L512" : 104,
98
+ "L1024" : 109
99
+ },
100
+ "dropped" : 0,
101
+ "latency_ms" : {
102
+ "p50" : 9.062875,
103
+ "p95" : 19.319541999999998
104
+ },
105
+ "max_probability_delta_vs_reference" : 0.034952796995639801,
106
+ "n" : 400,
107
+ "reference_argmax_agreement" : 1,
108
+ "reference_compared" : 400,
109
+ "state_truncated" : 7
110
+ },
111
+ "app.rag_relevance" : {
112
+ "accuracy" : 0.67500000000000004,
113
+ "buckets" : {
114
+ "L128" : 135,
115
+ "L256" : 262,
116
+ "L512" : 3
117
+ },
118
+ "dropped" : 0,
119
+ "latency_ms" : {
120
+ "p50" : 5.865875,
121
+ "p95" : 9.0795840000000005
122
+ },
123
+ "max_probability_delta_vs_reference" : 0.030605673789978027,
124
+ "n" : 400,
125
+ "reference_argmax_agreement" : 0.99250000000000005,
126
+ "reference_compared" : 400,
127
+ "state_truncated" : 0
128
+ },
129
+ "app.support_triage" : {
130
+ "accuracy" : 0.53749999999999998,
131
+ "buckets" : {
132
+ "L256" : 377,
133
+ "L512" : 23
134
+ },
135
+ "dropped" : 0,
136
+ "latency_ms" : {
137
+ "p50" : 5.2802090000000002,
138
+ "p95" : 8.903041
139
+ },
140
+ "max_probability_delta_vs_reference" : 0.052110105752944946,
141
+ "n" : 400,
142
+ "reference_argmax_agreement" : 0.99750000000000005,
143
+ "reference_compared" : 400,
144
+ "state_truncated" : 0
145
+ },
146
+ "jev.ag_news" : {
147
+ "accuracy" : 0.93500000000000005,
148
+ "buckets" : {
149
+ "L128" : 368,
150
+ "L256" : 32
151
+ },
152
+ "dropped" : 0,
153
+ "latency_ms" : {
154
+ "p50" : 4.5663749999999999,
155
+ "p95" : 11.922083000000001
156
+ },
157
+ "max_probability_delta_vs_reference" : 0.055396914482116699,
158
+ "n" : 400,
159
+ "reference_argmax_agreement" : 1,
160
+ "reference_compared" : 400,
161
+ "state_truncated" : 0
162
+ },
163
+ "jev.emotion" : {
164
+ "accuracy" : 0.53500000000000003,
165
+ "buckets" : {
166
+ "L128" : 400
167
+ },
168
+ "dropped" : 0,
169
+ "latency_ms" : {
170
+ "p50" : 3.8882919999999999,
171
+ "p95" : 4.2419169999999999
172
+ },
173
+ "max_probability_delta_vs_reference" : 0.086301475763320923,
174
+ "n" : 400,
175
+ "reference_argmax_agreement" : 0.995,
176
+ "reference_compared" : 400,
177
+ "state_truncated" : 0
178
+ },
179
+ "massive_intent.en" : {
180
+ "accuracy" : 0.65333333333333332,
181
+ "buckets" : {
182
+ "L256" : 300
183
+ },
184
+ "dropped" : 0,
185
+ "latency_ms" : {
186
+ "p50" : 5.5202499999999999,
187
+ "p95" : 6.7780420000000001
188
+ },
189
+ "max_probability_delta_vs_reference" : 0.14386069774627686,
190
+ "n" : 300,
191
+ "reference_argmax_agreement" : 0.98666666666666669,
192
+ "reference_compared" : 300,
193
+ "state_truncated" : 0
194
+ }
195
+ }
196
+ }
reports/verification-multilingual-L1024-e8.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_e8_L1024_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "1be66f2b54a08ea6a07f93c195777c4b105f7b2d9c09e47983ffecbb9f986f49",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "441cefaa5768572327ba89214566c6cb9a27aaacd479523b453f442c62f08eb2",
6
+ "Manifest.json": "3dba274c81ab7dc9e42a34cbf43ad6af5329692dc2d3c3e2ce21a78ee73cbea9"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 1024,
12
+ "max_options": 32,
13
+ "precision": "e8",
14
+ "package_bytes": 452793209,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 6.339008749986533,
34
+ "questions": 16,
35
+ "argmax_agreements": 16,
36
+ "max_probability_error": 0.005797207355499268,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 17.89439597632736,
40
+ "p95": 18.36130567244254,
41
+ "min": 17.64220802579075,
42
+ "max": 23.747125000227243
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.0025831088423728943,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 1.00006103515625,
60
+ "median_ms": 17.95933351968415
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.0004474017769098282,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 0.9998931884765625,
76
+ "median_ms": 17.946458974620327
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": true,
85
+ "coreml_argmax": 1,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.0013618916273117065,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 0.9998779296875,
92
+ "median_ms": 17.91014600894414
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.0008780807256698608,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 0.9998779296875,
108
+ "median_ms": 17.874291486805305
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.005797207355499268,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 0.999755859375,
124
+ "median_ms": 17.784750496502966
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": true,
133
+ "coreml_argmax": 1,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 5.608797073364258e-05,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 1.0000473260879517,
140
+ "median_ms": 18.000207986915484
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.002427428960800171,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 0.99993896484375,
156
+ "median_ms": 17.78575050411746
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 0.00012970063835382462,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 1.000091552734375,
172
+ "median_ms": 17.90124998660758
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": true,
181
+ "coreml_argmax": 0,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.00025540590286254883,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 1.00022292137146,
188
+ "median_ms": 17.889228998683393
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.0006077289581298828,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 0.999908447265625,
204
+ "median_ms": 17.80060399323702
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.00017899274826049805,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.0001795291900635,
220
+ "median_ms": 17.893854004796594
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.0004958007484674454,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 1.0000476837158203,
236
+ "median_ms": 17.79702099156566
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 157,
243
+ "options": 20,
244
+ "argmax_agrees": true,
245
+ "coreml_argmax": 3,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.0021624043583869934,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 0.99993896484375,
252
+ "median_ms": 17.815499508287758
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.004502624273300171,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 0.999755859375,
268
+ "median_ms": 18.24512452003546
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 174,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.0003440380096435547,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 0.9998931884765625,
284
+ "median_ms": 17.888771020807326
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 158,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.0009380578994750977,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 0.999786376953125,
300
+ "median_ms": 17.976166476728395
301
+ }
302
+ ],
303
+ "passed": true
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 11.53603254200425,
308
+ "questions": 16,
309
+ "argmax_agreements": 16,
310
+ "max_probability_error": 0.013535827398300171,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 80.1227499905508,
314
+ "p95": 81.49640440824443,
315
+ "min": 79.91241605486721,
316
+ "max": 92.36383298411965
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": true,
327
+ "coreml_argmax": 0,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.0022779330611228943,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 1.000244140625,
334
+ "median_ms": 80.32110400381498
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.0005542133003473282,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 0.999786376953125,
350
+ "median_ms": 80.10449950234033
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": true,
359
+ "coreml_argmax": 1,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.0025046318769454956,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 0.9996337890625,
366
+ "median_ms": 80.17289600684308
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.0018546432256698608,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 1.0001220703125,
382
+ "median_ms": 80.06041651242413
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.006285488605499268,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 0.99951171875,
398
+ "median_ms": 80.15079150209203
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": true,
407
+ "coreml_argmax": 1,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.0004321932792663574,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 0.9995424151420593,
414
+ "median_ms": 80.05002100253478
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": true,
423
+ "coreml_argmax": 1,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.0009625852108001709,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 0.9998779296875,
430
+ "median_ms": 80.20041699637659
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": true,
439
+ "coreml_argmax": 0,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 9.155366569757462e-05,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 1.0000534057617188,
446
+ "median_ms": 80.10068751173094
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": true,
455
+ "coreml_argmax": 0,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.00023287534713745117,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.9997386932373047,
462
+ "median_ms": 80.0608120043762
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.0005619749426841736,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 1.0004425048828125,
478
+ "median_ms": 81.02227098424919
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.00017899274826049805,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.0001797676086426,
494
+ "median_ms": 80.29522901051678
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": true,
503
+ "coreml_argmax": 0,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.0009060502052307129,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 1.0005512237548828,
510
+ "median_ms": 80.08233297732659
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 157,
517
+ "options": 20,
518
+ "argmax_agrees": true,
519
+ "coreml_argmax": 3,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.0027727559208869934,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 1.00018310546875,
526
+ "median_ms": 80.27691699680872
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.013535827398300171,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 0.99951171875,
542
+ "median_ms": 80.1208334742114
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 174,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.0003440380096435547,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 0.999847412109375,
558
+ "median_ms": 80.04837500629947
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 158,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.0009380578994750977,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 0.9996337890625,
574
+ "median_ms": 80.1351044792682
575
+ }
576
+ ],
577
+ "passed": true
578
+ }
579
+ },
580
+ "passed": true
581
+ }
reports/verification-multilingual-L128-e8.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_e8_L128_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "a702d6e956cfecd233b8e9573c2bb3a018c71c3fcac29d60868e5b01e7a6a6d3",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "ca1e5da71f1498eac7b68722c9f5c3d43d1ce299aaf0c29743585749f3cc9677",
6
+ "Manifest.json": "fff2cdff70403a2b33aff04c4b93ba1d443873e796c3cd72cc8f59a82b8d96bc"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 128,
12
+ "max_options": 32,
13
+ "precision": "e8",
14
+ "package_bytes": 448435058,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 6.075320333999116,
34
+ "questions": 16,
35
+ "argmax_agreements": 16,
36
+ "max_probability_error": 0.015331089496612549,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 3.8387505046557635,
40
+ "p95": 4.020178844803013,
41
+ "min": 3.719458996783942,
42
+ "max": 4.489124985411763
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.0025831088423728943,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 1.00006103515625,
60
+ "median_ms": 3.8522914983332157
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.0004168841987848282,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 0.9999237060546875,
76
+ "median_ms": 3.8293334946502
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": true,
85
+ "coreml_argmax": 1,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.0018501728773117065,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 0.9998779296875,
92
+ "median_ms": 3.821459016762674
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.0011222213506698608,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 1.0,
108
+ "median_ms": 3.8383124920073897
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.006026566028594971,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 0.9998779296875,
124
+ "median_ms": 3.847000014502555
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": true,
133
+ "coreml_argmax": 1,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 5.608797073364258e-05,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 1.0000249147415161,
140
+ "median_ms": 3.7663340044673532
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.001695007085800171,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 1.0001220703125,
156
+ "median_ms": 3.8036454934626818
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 0.00012970063835382462,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 1.000091552734375,
172
+ "median_ms": 3.8716669951099902
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": true,
181
+ "coreml_argmax": 0,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.00025540590286254883,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 1.0002155303955078,
188
+ "median_ms": 3.9035830122884363
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.0003025755286216736,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 1.00018310546875,
204
+ "median_ms": 3.8823539798613638
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.00017899274826049805,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.0001797676086426,
220
+ "median_ms": 3.782583517022431
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.0004576537758111954,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 1.0001220703125,
236
+ "median_ms": 3.77122953068465
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 128,
243
+ "options": 20,
244
+ "argmax_agrees": true,
245
+ "coreml_argmax": 3,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.015331089496612549,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 1.0000648498535156,
252
+ "median_ms": 3.791604482103139
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.004014343023300171,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 0.999755859375,
268
+ "median_ms": 3.802395978709683
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 128,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.00012070499360561371,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 1.000091552734375,
284
+ "median_ms": 3.8319380255416036
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 128,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.004284918308258057,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 0.9997673034667969,
300
+ "median_ms": 3.7830204819329083
301
+ }
302
+ ],
303
+ "passed": true
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 8.86458433396183,
308
+ "questions": 16,
309
+ "argmax_agreements": 16,
310
+ "max_probability_error": 0.014354526996612549,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 3.638145513832569,
314
+ "p95": 3.827884842758067,
315
+ "min": 3.580958000384271,
316
+ "max": 626.2272089952603
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": true,
327
+ "coreml_argmax": 0,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.0022779330611228943,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 1.000244140625,
334
+ "median_ms": 3.7582709919661283
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.0005542133003473282,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 0.999786376953125,
350
+ "median_ms": 3.6189999955240637
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": true,
359
+ "coreml_argmax": 1,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.0025046318769454956,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 0.9996337890625,
366
+ "median_ms": 3.6314789904281497
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.0018546432256698608,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 1.0001220703125,
382
+ "median_ms": 3.6308334674686193
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.006285488605499268,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 0.99951171875,
398
+ "median_ms": 3.6329374997876585
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": true,
407
+ "coreml_argmax": 1,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.0004321932792663574,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 0.9995424151420593,
414
+ "median_ms": 3.7671044701710343
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": true,
423
+ "coreml_argmax": 1,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.0009625852108001709,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 0.9998779296875,
430
+ "median_ms": 3.6246664822101593
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": true,
439
+ "coreml_argmax": 0,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 9.155366569757462e-05,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 1.0000534057617188,
446
+ "median_ms": 3.6396670329850167
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": true,
455
+ "coreml_argmax": 0,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.00023287534713745117,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.9997386932373047,
462
+ "median_ms": 3.6316874902695417
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.0005619749426841736,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 1.0004425048828125,
478
+ "median_ms": 3.7328749895095825
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.00017899274826049805,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.0001797676086426,
494
+ "median_ms": 3.6238130123820156
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": true,
503
+ "coreml_argmax": 0,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.0009060502052307129,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 1.0005512237548828,
510
+ "median_ms": 3.632687497884035
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 128,
517
+ "options": 20,
518
+ "argmax_agrees": true,
519
+ "coreml_argmax": 3,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.014354526996612549,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 0.9998035430908203,
526
+ "median_ms": 3.6104585451539606
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.013535827398300171,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 0.99951171875,
542
+ "median_ms": 3.628916514571756
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 128,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 5.966983735561371e-05,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 1.000030517578125,
558
+ "median_ms": 3.823729493888095
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 128,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.004299595952033997,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 0.9996833801269531,
574
+ "median_ms": 3.7571460125036538
575
+ }
576
+ ],
577
+ "passed": true
578
+ }
579
+ },
580
+ "passed": true
581
+ }
reports/verification-multilingual-L128-w4.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_w4_L128_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "c63a2e91dca84518e5dfac91b506daa42d35e0577b0fc7db93538b813f99d3fa",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "573c65383a9b8ded8f6c5bb0e7f10b137cdb454cd6e0ccc3a5a1e2f5611c52af",
6
+ "Manifest.json": "e5d5e20a35f0d1d94974834915b607c1eee46cdb91ad7a350f96926ca0255263"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 128,
12
+ "max_options": 32,
13
+ "precision": "w4",
14
+ "package_bytes": 456441713,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 1.0729787919553928,
34
+ "questions": 16,
35
+ "argmax_agreements": 12,
36
+ "max_probability_error": 0.7274829745292664,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 3.9001669792924076,
40
+ "p95": 4.311243767733686,
41
+ "min": 3.7357090041041374,
42
+ "max": 5.792083975393325
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.0635572075843811,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 0.9998779296875,
60
+ "median_ms": 4.182292002951726
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.15346801280975342,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 0.9998779296875,
76
+ "median_ms": 3.9166875067166984
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": false,
85
+ "coreml_argmax": 0,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.09560063481330872,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 1.0001220703125,
92
+ "median_ms": 3.966332966228947
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.08459603786468506,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 1.0001220703125,
108
+ "median_ms": 3.85877100052312
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.15770003199577332,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 1.000244140625,
124
+ "median_ms": 3.7997290201019496
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": false,
133
+ "coreml_argmax": 0,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 0.7274829745292664,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 0.99993896484375,
140
+ "median_ms": 3.8310835079755634
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.27412307262420654,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 0.9998779296875,
156
+ "median_ms": 3.865125501761213
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 0.055702269077301025,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 0.999908447265625,
172
+ "median_ms": 3.889250016072765
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": false,
181
+ "coreml_argmax": 1,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.6696664690971375,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 1.0001220703125,
188
+ "median_ms": 4.258249478880316
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.043149255216121674,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 1.00006103515625,
204
+ "median_ms": 3.951812512241304
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.1131022572517395,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 0.999908447265625,
220
+ "median_ms": 3.922499978216365
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.3140353560447693,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 1.0001220703125,
236
+ "median_ms": 3.9491454954259098
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 128,
243
+ "options": 20,
244
+ "argmax_agrees": false,
245
+ "coreml_argmax": 0,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.42387789487838745,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 0.9999847412109375,
252
+ "median_ms": 3.9071874925866723
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.18882879614830017,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 1.0,
268
+ "median_ms": 3.856478986563161
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 128,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.15432602167129517,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 0.9998779296875,
284
+ "median_ms": 3.878874995280057
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 128,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.030788853764533997,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 1.000091552734375,
300
+ "median_ms": 3.8517920474987477
301
+ }
302
+ ],
303
+ "passed": false
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 0.05808512499788776,
308
+ "questions": 16,
309
+ "argmax_agreements": 12,
310
+ "max_probability_error": 0.7284595370292664,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 3.720854001585394,
314
+ "p95": 3.8386354368412867,
315
+ "min": 3.6097500124014914,
316
+ "max": 4.019500047434121
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": true,
327
+ "coreml_argmax": 0,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.0650220513343811,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 0.999755859375,
334
+ "median_ms": 3.7071249971631914
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.15420041978359222,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 1.000244140625,
350
+ "median_ms": 3.79056204110384
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": false,
359
+ "coreml_argmax": 0,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.09523442387580872,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 0.999755859375,
366
+ "median_ms": 3.764145978493616
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.08459603786468506,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 0.9996337890625,
382
+ "median_ms": 3.7676040083169937
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.15599104762077332,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 0.999755859375,
398
+ "median_ms": 3.7193335010670125
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": false,
407
+ "coreml_argmax": 0,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.7284595370292664,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 0.99951171875,
414
+ "median_ms": 3.682729002321139
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": true,
423
+ "coreml_argmax": 1,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.27216994762420654,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 1.00048828125,
430
+ "median_ms": 3.704917005961761
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": true,
439
+ "coreml_argmax": 0,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 0.055702269077301025,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 0.99981689453125,
446
+ "median_ms": 3.703791997395456
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": false,
455
+ "coreml_argmax": 1,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.6625863909721375,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 1.00018310546875,
462
+ "median_ms": 3.665396012365818
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.04259991645812988,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 0.9998779296875,
478
+ "median_ms": 3.712687495863065
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.1179850697517395,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 0.99969482421875,
494
+ "median_ms": 3.6586874921340495
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": true,
503
+ "coreml_argmax": 0,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.3145236372947693,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 0.9998779296875,
510
+ "median_ms": 3.660646005300805
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 128,
517
+ "options": 20,
518
+ "argmax_agrees": false,
519
+ "coreml_argmax": 0,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.42399996519088745,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 0.998809814453125,
526
+ "median_ms": 3.7595834874082357
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.18785223364830017,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 0.99951171875,
542
+ "median_ms": 3.777874488150701
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 128,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.15481430292129517,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 0.9998779296875,
558
+ "median_ms": 3.7151249998714775
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 128,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.033504918217659,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 1.0006256103515625,
574
+ "median_ms": 3.673125000204891
575
+ }
576
+ ],
577
+ "passed": false
578
+ }
579
+ },
580
+ "passed": false
581
+ }
reports/verification-multilingual-L128-w4e8.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_w4e8_L128_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "5c0ee3881561d52854c9ff3c094713cac4797f056ef7162021ea3f1b5614979b",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "98c39cc41bbf495cc933648522fb345709b4062822c3322fefba9d3e8c5c40aa",
6
+ "Manifest.json": "fb608932bd9ea1b5c41621f5aa67768ad8e98f812de50f010e946d063fd871ee"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 128,
12
+ "max_options": 32,
13
+ "precision": "w4e8",
14
+ "package_bytes": 260602440,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 5.722255375003442,
34
+ "questions": 16,
35
+ "argmax_agreements": 11,
36
+ "max_probability_error": 0.7338754534721375,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 4.154979513259605,
40
+ "p95": 5.602722565527074,
41
+ "min": 3.771624993532896,
42
+ "max": 7.908458006568253
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.049885354936122894,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 1.0,
60
+ "median_ms": 3.993958031060174
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.16909301280975342,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 1.0,
76
+ "median_ms": 4.158916504820809
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": false,
85
+ "coreml_argmax": 0,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.10023930668830872,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 1.0001220703125,
92
+ "median_ms": 4.195916495518759
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.08752572536468506,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 1.0001220703125,
108
+ "median_ms": 4.104916530195624
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": false,
117
+ "coreml_argmax": 1,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.18113753199577332,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 1.0,
124
+ "median_ms": 4.256458021700382
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": false,
133
+ "coreml_argmax": 0,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 0.7035571932792664,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 0.9998779296875,
140
+ "median_ms": 4.523917014012113
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.24116408824920654,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 1.0001220703125,
156
+ "median_ms": 4.675916512496769
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 0.07236480712890625,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 1.00006103515625,
172
+ "median_ms": 5.404833500506356
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": false,
181
+ "coreml_argmax": 1,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.7338754534721375,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 1.0,
188
+ "median_ms": 4.865312483161688
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.038815759122371674,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 1.0001220703125,
204
+ "median_ms": 4.0588130068499595
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.0989421010017395,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.0001220703125,
220
+ "median_ms": 4.04439601697959
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.3438205122947693,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 0.9998779296875,
236
+ "median_ms": 4.084146028617397
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 128,
243
+ "options": 20,
244
+ "argmax_agrees": false,
245
+ "coreml_argmax": 0,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.42412203550338745,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 0.9999847412109375,
252
+ "median_ms": 3.9592709799762815
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.17125067114830017,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 1.0,
268
+ "median_ms": 3.90997898648493
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 128,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.16909652948379517,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 1.0001220703125,
284
+ "median_ms": 4.13027050672099
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 128,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.05040258169174194,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 1.000213623046875,
300
+ "median_ms": 3.8169795006979257
301
+ }
302
+ ],
303
+ "passed": false
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 8.592580166994594,
308
+ "questions": 16,
309
+ "argmax_agreements": 11,
310
+ "max_probability_error": 0.7287485003471375,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 3.7338540132623166,
314
+ "p95": 4.254380951169878,
315
+ "min": 3.609207982663065,
316
+ "max": 5.975832988042384
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": true,
327
+ "coreml_argmax": 0,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.050861917436122894,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 1.00048828125,
334
+ "median_ms": 3.767687507206574
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.16958129405975342,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 1.0,
350
+ "median_ms": 3.648437501396984
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": false,
359
+ "coreml_argmax": 0,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.09975102543830872,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 0.9998779296875,
366
+ "median_ms": 3.6624585336539894
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.08752572536468506,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 1.0,
382
+ "median_ms": 3.6646875087171793
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": false,
391
+ "coreml_argmax": 1,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.17942854762077332,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 0.999755859375,
398
+ "median_ms": 3.6670624976977706
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": false,
407
+ "coreml_argmax": 0,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.7045337557792664,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 1.00048828125,
414
+ "median_ms": 3.6677290045190603
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": true,
423
+ "coreml_argmax": 1,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.24067580699920654,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 1.000244140625,
430
+ "median_ms": 3.6528539785649627
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": true,
439
+ "coreml_argmax": 0,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 0.07169342041015625,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 1.0003662109375,
446
+ "median_ms": 3.706833493197337
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": false,
455
+ "coreml_argmax": 1,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.7287485003471375,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.9998779296875,
462
+ "median_ms": 3.679521003505215
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.039304040372371674,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 1.0001220703125,
478
+ "median_ms": 3.8889585121069103
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.1013835072517395,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.000091552734375,
494
+ "median_ms": 3.7854375259485096
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": true,
503
+ "coreml_argmax": 0,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.3467501997947693,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 1.0001220703125,
510
+ "median_ms": 3.774520999286324
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 128,
517
+ "options": 20,
518
+ "argmax_agrees": false,
519
+ "coreml_argmax": 0,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.42216891050338745,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 0.99951171875,
526
+ "median_ms": 4.269541997928172
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.16880923509597778,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 1.000244140625,
542
+ "median_ms": 4.042041546199471
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 128,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.16848617792129517,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 1.00048828125,
558
+ "median_ms": 3.7245624989736825
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 128,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.05040258169174194,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 0.999664306640625,
574
+ "median_ms": 3.689062490593642
575
+ }
576
+ ],
577
+ "passed": false
578
+ }
579
+ },
580
+ "passed": false
581
+ }
reports/verification-multilingual-L128-w6.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_w6_L128_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "437fcb102aa75b0b4298d6884594f3a633843b40634e72d8cefeabdf092ffb5f",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "2b9ba8dd6c102d1982b66fe9c18b15f155a1231fa299ebc3659d7f799e087e14",
6
+ "Manifest.json": "3ca5e9231730455da0b2ab6c913ad8f722018cd6e18a10e961fd3eebe8a26178"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 128,
12
+ "max_options": 32,
13
+ "precision": "w6",
14
+ "package_bytes": 487758065,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 0.40539787500165403,
34
+ "questions": 16,
35
+ "argmax_agreements": 15,
36
+ "max_probability_error": 0.11412176489830017,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 4.023020999738947,
40
+ "p95": 4.896181236836128,
41
+ "min": 3.7430000375024974,
42
+ "max": 6.122291961219162
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.0023607611656188965,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 1.000244140625,
60
+ "median_ms": 4.197187488898635
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.002100825309753418,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 0.9999847412109375,
76
+ "median_ms": 4.224250471452251
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": false,
85
+ "coreml_argmax": 0,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.06976586580276489,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 0.9998779296875,
92
+ "median_ms": 4.11027148948051
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.05334603786468506,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 1.0,
108
+ "median_ms": 4.552416998194531
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.003983199596405029,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 0.9998779296875,
124
+ "median_ms": 4.148854000959545
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": true,
133
+ "coreml_argmax": 1,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 0.0002187361242249608,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 0.9998368620872498,
140
+ "median_ms": 4.186771489912644
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.00848400592803955,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 1.0,
156
+ "median_ms": 4.011520504718646
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 0.00017547700554132462,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 1.0001373291015625,
172
+ "median_ms": 3.8826040108688176
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": true,
181
+ "coreml_argmax": 0,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.00023287534713745117,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 0.999935507774353,
188
+ "median_ms": 3.886792022967711
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.015897057950496674,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 1.000152587890625,
204
+ "median_ms": 3.8653119991067797
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.00017899274826049805,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.0001966953277588,
220
+ "median_ms": 3.8469999853987247
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.0015353560447692871,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 0.9997730255126953,
236
+ "median_ms": 3.8703329919371754
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 128,
243
+ "options": 20,
244
+ "argmax_agrees": true,
245
+ "coreml_argmax": 3,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.06206148862838745,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 1.0001564025878906,
252
+ "median_ms": 3.8793960120528936
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.11412176489830017,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 1.0,
268
+ "median_ms": 4.9398334813304245
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 128,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.004853665828704834,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 1.00006103515625,
284
+ "median_ms": 3.991229517851025
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 128,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.03331273794174194,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 1.0002517700195312,
300
+ "median_ms": 3.9200414903461933
301
+ }
302
+ ],
303
+ "passed": false
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 0.06528987496858463,
308
+ "questions": 16,
309
+ "argmax_agreements": 15,
310
+ "max_probability_error": 0.11753970384597778,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 3.664499963633716,
314
+ "p95": 3.813516642549075,
315
+ "min": 3.6027920432388783,
316
+ "max": 4.030583018902689
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": true,
327
+ "coreml_argmax": 0,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.0013841986656188965,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 1.000244140625,
334
+ "median_ms": 3.6517085100058466
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.002589106559753418,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 0.9996337890625,
350
+ "median_ms": 3.6495624808594584
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": false,
359
+ "coreml_argmax": 0,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.07001000642776489,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 0.9996337890625,
366
+ "median_ms": 3.6430624895729125
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.05334603786468506,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 0.999755859375,
382
+ "median_ms": 3.673332976177335
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.007645308971405029,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 1.0,
398
+ "median_ms": 3.665978991193697
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": true,
407
+ "coreml_argmax": 1,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.0005443692207336426,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 1.0003440380096436,
414
+ "median_ms": 3.6652290145866573
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": true,
423
+ "coreml_argmax": 1,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.006042599678039551,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 1.00006103515625,
430
+ "median_ms": 3.835354495095089
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": true,
439
+ "coreml_argmax": 0,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 9.155366569757462e-05,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 1.0000534057617188,
446
+ "median_ms": 3.758832986932248
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": true,
455
+ "coreml_argmax": 0,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.00023287534713745117,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.9999407529830933,
462
+ "median_ms": 3.7352295184973627
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.016965173184871674,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 1.000732421875,
478
+ "median_ms": 3.6477295216172934
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.00017899274826049805,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.0001966953277588,
494
+ "median_ms": 3.6439785035327077
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": true,
503
+ "coreml_argmax": 0,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.0012475159019231796,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 1.0002479553222656,
510
+ "median_ms": 3.663499985123053
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 128,
517
+ "options": 20,
518
+ "argmax_agrees": true,
519
+ "coreml_argmax": 3,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.06425875425338745,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 0.9998111724853516,
526
+ "median_ms": 3.671749960631132
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.11753970384597778,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 1.0003662109375,
542
+ "median_ms": 3.6952500231564045
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 128,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.004853665828704834,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 1.000274658203125,
558
+ "median_ms": 3.651062026619911
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 128,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.03282445669174194,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 0.9995269775390625,
574
+ "median_ms": 3.650186990853399
575
+ }
576
+ ],
577
+ "passed": false
578
+ }
579
+ },
580
+ "passed": false
581
+ }
reports/verification-multilingual-L128-w6e8.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_w6e8_L128_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "329ca89bebd8fe69df5445421c4c7efd64247391f5d6c7aa145b67d30385cb15",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "f576015e91ea145b1268168b6dc0c6ac97a5bbfd6923ee1620cb3a38a6525be9",
6
+ "Manifest.json": "ef4c5d2e529ff4bba0d39f9509fc4895c5a6f18dce9a11c30688890021eca773"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 128,
12
+ "max_options": 32,
13
+ "precision": "w6e8",
14
+ "package_bytes": 291918867,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 6.012676417012699,
34
+ "questions": 16,
35
+ "argmax_agreements": 15,
36
+ "max_probability_error": 0.11705142259597778,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 4.070562485139817,
40
+ "p95": 4.799069106229581,
41
+ "min": 3.748167015146464,
42
+ "max": 6.246125034522265
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.0006299838423728943,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 1.00006103515625,
60
+ "median_ms": 3.9375209889840335
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.0012615825980901718,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 1.0001373291015625,
76
+ "median_ms": 3.934312961064279
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": false,
85
+ "coreml_argmax": 0,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.06732445955276489,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 0.999755859375,
92
+ "median_ms": 3.815457981545478
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.04846322536468506,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 1.0,
108
+ "median_ms": 4.293645499274135
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.007645308971405029,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 0.9998779296875,
124
+ "median_ms": 3.9634164713788778
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": true,
133
+ "coreml_argmax": 1,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 0.0002473463537171483,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 0.9998081922531128,
140
+ "median_ms": 4.605561989592388
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.01239025592803955,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 1.00006103515625,
156
+ "median_ms": 4.601292021106929
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 9.536836296319962e-05,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 1.0000572204589844,
172
+ "median_ms": 4.162083467235789
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": true,
181
+ "coreml_argmax": 0,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.00025540590286254883,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 1.0002191066741943,
188
+ "median_ms": 4.251854494214058
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.014951013028621674,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 1.00018310546875,
204
+ "median_ms": 4.1342704789713025
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.00017899274826049805,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.0002022981643677,
220
+ "median_ms": 4.035437508719042
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.0007172729820013046,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 1.0001888275146484,
236
+ "median_ms": 3.949687525164336
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 128,
243
+ "options": 20,
244
+ "argmax_agrees": true,
245
+ "coreml_argmax": 3,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.026790112257003784,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 0.9998741149902344,
252
+ "median_ms": 4.269541503163055
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.11705142259597778,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 1.000244140625,
268
+ "median_ms": 4.126771003939211
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 128,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.004853665828704834,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 1.000030517578125,
284
+ "median_ms": 3.9383749826811254
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 128,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.025011956691741943,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 0.9998321533203125,
300
+ "median_ms": 3.8940419908612967
301
+ }
302
+ ],
303
+ "passed": false
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 8.698386332951486,
308
+ "questions": 16,
309
+ "argmax_agreements": 15,
310
+ "max_probability_error": 0.11998111009597778,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 3.672833467135206,
314
+ "p95": 3.8521848036907613,
315
+ "min": 3.614499990362674,
316
+ "max": 4.668750043492764
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": true,
327
+ "coreml_argmax": 0,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.0005689263343811035,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 0.99981689453125,
334
+ "median_ms": 3.8271665398497134
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.001612544059753418,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 0.999847412109375,
350
+ "median_ms": 3.7480210012290627
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": false,
359
+ "coreml_argmax": 0,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.06854516267776489,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 1.0,
366
+ "median_ms": 3.646500001195818
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.04846322536468506,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 0.9993896484375,
382
+ "median_ms": 3.6573539837263525
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.01252812147140503,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 0.9998779296875,
398
+ "median_ms": 3.671353973913938
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": true,
407
+ "coreml_argmax": 1,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.0005443692207336426,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 1.0002832412719727,
414
+ "median_ms": 3.661396011011675
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": true,
423
+ "coreml_argmax": 1,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.00994884967803955,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 0.999755859375,
430
+ "median_ms": 3.6549170035868883
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": true,
439
+ "coreml_argmax": 0,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 0.00011062715202569962,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 1.0000724792480469,
446
+ "median_ms": 3.662938019260764
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": true,
455
+ "coreml_argmax": 0,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.00023287534713745117,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.999731183052063,
462
+ "median_ms": 3.7158330087549984
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.015744447708129883,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 0.999481201171875,
478
+ "median_ms": 3.6696250026579946
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.00017899274826049805,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.0002055168151855,
494
+ "median_ms": 3.810978989349678
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": true,
503
+ "coreml_argmax": 0,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.0010470747947692871,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 0.9998512268066406,
510
+ "median_ms": 3.7581874639727175
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 128,
517
+ "options": 20,
518
+ "argmax_agrees": true,
519
+ "coreml_argmax": 3,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.023860424757003784,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 1.000925064086914,
526
+ "median_ms": 3.732874523848295
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.11998111009597778,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 1.0001220703125,
542
+ "median_ms": 3.666708478704095
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 128,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.004929954186081886,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 0.9999237060546875,
558
+ "median_ms": 3.648000507382676
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 128,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.021105706691741943,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 1.0004081726074219,
574
+ "median_ms": 3.6529169883579016
575
+ }
576
+ ],
577
+ "passed": false
578
+ }
579
+ },
580
+ "passed": false
581
+ }
reports/verification-multilingual-L128-w8.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_w8_L128_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "4d1358ac20b62ec9aa90d76b3ce22e04f76505d3fbc2242f9ea83a47527f709d",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "7c0bf0c97c3dfa524571c349f24241d1499514c90f5bef5bfccceb7c1c3cfee4",
6
+ "Manifest.json": "4bdc36727345b7bf6c23248407c45a0d5d466b20170506b5ef9c34273b55a68e"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 128,
12
+ "max_options": 32,
13
+ "precision": "w8",
14
+ "package_bytes": 323530193,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 0.30971424997551367,
34
+ "questions": 16,
35
+ "argmax_agreements": 16,
36
+ "max_probability_error": 0.04169827699661255,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 4.134541988605633,
40
+ "p95": 4.69923158234451,
41
+ "min": 3.8557079969905317,
42
+ "max": 5.924416007474065
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.0025831088423728943,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 1.00006103515625,
60
+ "median_ms": 4.497124988120049
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.00016294978559017181,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 1.0000152587890625,
76
+ "median_ms": 4.4412079732865095
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": true,
85
+ "coreml_argmax": 1,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.004824459552764893,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 0.9998779296875,
92
+ "median_ms": 4.317542014177889
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.005494475364685059,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 0.9998779296875,
108
+ "median_ms": 4.138145508477464
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.025696933269500732,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 1.0001220703125,
124
+ "median_ms": 4.157624993240461
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": true,
133
+ "coreml_argmax": 1,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 0.0001596083166077733,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 0.9998961687088013,
140
+ "median_ms": 4.174895962933078
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.010239928960800171,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 1.00006103515625,
156
+ "median_ms": 4.044833505759016
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 0.0005264878273010254,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 0.9998092651367188,
172
+ "median_ms": 4.042041982756928
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": true,
181
+ "coreml_argmax": 0,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.00025540590286254883,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 1.0001870393753052,
188
+ "median_ms": 4.069791495567188
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.0013606324791908264,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 0.9999847412109375,
204
+ "median_ms": 4.184874997008592
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.00017899274826049805,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.0001893043518066,
220
+ "median_ms": 4.190374515019357
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.0013943314552307129,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 1.0001106262207031,
236
+ "median_ms": 4.013374971691519
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 128,
243
+ "options": 20,
244
+ "argmax_agrees": true,
245
+ "coreml_argmax": 3,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.04169827699661255,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 1.0001888275146484,
252
+ "median_ms": 4.021686996566132
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.02745184302330017,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 0.999755859375,
268
+ "median_ms": 3.948749479604885
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 128,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.001435697078704834,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 1.00018310546875,
284
+ "median_ms": 4.0304374997504056
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 128,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.021206334233283997,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 1.000213623046875,
300
+ "median_ms": 4.040937987156212
301
+ }
302
+ ],
303
+ "passed": false
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 0.058956374996341765,
308
+ "questions": 16,
309
+ "argmax_agreements": 11,
310
+ "max_probability_error": 0.6675509810447693,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 3.8277290004771203,
314
+ "p95": 3.96077292971313,
315
+ "min": 3.718333027791232,
316
+ "max": 5.271999980323017
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": false,
327
+ "coreml_argmax": 1,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.4746900498867035,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 1.0,
334
+ "median_ms": 3.7765210145153105
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.17495238780975342,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 0.999755859375,
350
+ "median_ms": 3.7898749869782478
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": false,
359
+ "coreml_argmax": 2,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.0824563056230545,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 1.0,
366
+ "median_ms": 3.790062037296593
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.22170449793338776,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 1.0003662109375,
382
+ "median_ms": 3.7895625282544643
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.14325064420700073,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 0.9996337890625,
398
+ "median_ms": 3.744395973626524
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": true,
407
+ "coreml_argmax": 1,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.6139575839042664,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 0.9991455078125,
414
+ "median_ms": 3.7628124991897494
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": false,
423
+ "coreml_argmax": 2,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.29144299030303955,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 1.000244140625,
430
+ "median_ms": 3.837042022496462
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": false,
439
+ "coreml_argmax": 1,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 0.533485472202301,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 0.999755859375,
446
+ "median_ms": 3.8239584828261286
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": true,
455
+ "coreml_argmax": 0,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.00023287534713745117,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.9997035264968872,
462
+ "median_ms": 3.8753960106987506
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.23302960395812988,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 0.99951171875,
478
+ "median_ms": 3.8337705191224813
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.5684245228767395,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.000244140625,
494
+ "median_ms": 3.8206460303626955
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": false,
503
+ "coreml_argmax": 2,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.6675509810447693,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 0.99951171875,
510
+ "median_ms": 3.8565624854527414
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 128,
517
+ "options": 20,
518
+ "argmax_agrees": true,
519
+ "coreml_argmax": 3,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.33147066831588745,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 1.0,
526
+ "median_ms": 3.840166493318975
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.04869207739830017,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 1.0,
542
+ "median_ms": 3.8751665269955993
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 128,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.43752914667129517,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 0.99951171875,
558
+ "median_ms": 3.910416504368186
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 128,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.10606664419174194,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 0.99969482421875,
574
+ "median_ms": 3.908667014911771
575
+ }
576
+ ],
577
+ "passed": false
578
+ }
579
+ },
580
+ "passed": false
581
+ }
reports/verification-multilingual-L128-w8e.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_w8e_L128_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "6b64793cb28ca42ff8fb485ac9f4d3fffff6b83f7154ad74c833651697e70704",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "7e8f717e9ae24f9e7849ae6411df0d20b4d1e4d2090511597210f3e59652b8f1",
6
+ "Manifest.json": "cc4e600199e5f1220eaf701ccfd436c0ff191219155fdb84182f9442dc498289"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 128,
12
+ "max_options": 32,
13
+ "precision": "w8e",
14
+ "package_bytes": 519369445,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 0.3818642499973066,
34
+ "questions": 16,
35
+ "argmax_agreements": 16,
36
+ "max_probability_error": 0.029359042644500732,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 4.109186993446201,
40
+ "p95": 7.231489886180504,
41
+ "min": 3.7559160264208913,
42
+ "max": 29.032292019110173
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 8.066743612289429e-05,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 1.0,
60
+ "median_ms": 4.101333499420434
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.0006817486137151718,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 1.0000457763671875,
76
+ "median_ms": 4.100457997992635
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": true,
85
+ "coreml_argmax": 1,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.005801022052764893,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 1.0,
92
+ "median_ms": 3.924895980162546
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.005982756614685059,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 0.9998779296875,
108
+ "median_ms": 3.843708516797051
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.029359042644500732,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 1.0001220703125,
124
+ "median_ms": 8.767875522607937
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": true,
133
+ "coreml_argmax": 1,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 0.00013147492427378893,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 0.99992436170578,
140
+ "median_ms": 4.394833522383124
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.009995788335800171,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 0.99981689453125,
156
+ "median_ms": 3.9516875112894922
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 0.00015640351921319962,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 1.0001182556152344,
172
+ "median_ms": 4.1966664721257985
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": true,
181
+ "coreml_argmax": 0,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.00025540590286254883,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 1.000189185142517,
188
+ "median_ms": 4.051833035191521
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.0020015016198158264,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 0.9998321533203125,
204
+ "median_ms": 4.216625005938113
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.00017899274826049805,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.000185251235962,
220
+ "median_ms": 4.073020973009989
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.0009060502052307129,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 0.9998893737792969,
236
+ "median_ms": 4.18681200244464
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 128,
243
+ "options": 20,
244
+ "argmax_agrees": true,
245
+ "coreml_argmax": 3,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.02070218324661255,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 0.9997787475585938,
252
+ "median_ms": 4.019979474833235
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.02378973364830017,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 1.0,
268
+ "median_ms": 4.726208513602614
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 128,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.0010694805532693863,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 0.9998779296875,
284
+ "median_ms": 4.220062546664849
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 128,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.015346959233283997,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 1.0001373291015625,
300
+ "median_ms": 4.182229022262618
301
+ }
302
+ ],
303
+ "passed": false
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 0.05620520899537951,
308
+ "questions": 16,
309
+ "argmax_agreements": 11,
310
+ "max_probability_error": 0.6655978560447693,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 3.6921665305271745,
314
+ "p95": 3.90408328094054,
315
+ "min": 3.6110419896431267,
316
+ "max": 6.501417024992406
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": false,
327
+ "coreml_argmax": 1,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.4751783013343811,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 0.99951171875,
334
+ "median_ms": 3.7319585098885
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.17641723155975342,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 0.9993896484375,
350
+ "median_ms": 3.688833530759439
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": false,
359
+ "coreml_argmax": 2,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.0827004462480545,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 1.0006103515625,
366
+ "median_ms": 3.7430415104608983
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.22194863855838776,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 1.0001220703125,
382
+ "median_ms": 3.673228988191113
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.14068716764450073,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 0.9998779296875,
398
+ "median_ms": 3.666521020932123
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": true,
407
+ "coreml_argmax": 1,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.6146900057792664,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 1.00048828125,
414
+ "median_ms": 3.6692084977403283
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": false,
423
+ "coreml_argmax": 2,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.28802502155303955,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 0.99951171875,
430
+ "median_ms": 3.764437511563301
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": false,
439
+ "coreml_argmax": 1,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 0.5356826782226562,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 1.000244140625,
446
+ "median_ms": 3.9228539972100407
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": true,
455
+ "coreml_argmax": 0,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.00023287534713745117,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.9997237324714661,
462
+ "median_ms": 3.771416493691504
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.23254135251045227,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 1.0,
478
+ "median_ms": 3.784479486057535
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.5684245228767395,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.0,
494
+ "median_ms": 3.7379164714366198
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": false,
503
+ "coreml_argmax": 2,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.6655978560447693,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 1.000244140625,
510
+ "median_ms": 3.66466652485542
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 128,
517
+ "options": 20,
518
+ "argmax_agrees": true,
519
+ "coreml_argmax": 3,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.34282320737838745,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 0.99945068359375,
526
+ "median_ms": 3.648666985100135
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.04918032884597778,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 1.000244140625,
542
+ "median_ms": 3.6654994764830917
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 128,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.43704086542129517,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 0.999755859375,
558
+ "median_ms": 3.6537709820549935
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 128,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.10411351919174194,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 1.000244140625,
574
+ "median_ms": 3.6801249952986836
575
+ }
576
+ ],
577
+ "passed": false
578
+ }
579
+ },
580
+ "passed": false
581
+ }
reports/verification-multilingual-L128-w8enc.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_w8enc_L128_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "5879590bd22681a203478520fa46233185dc14bead3c7fb3c542ca50cb7f46f3",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "badc17e87d0f012690822536e23a99420ea255e3315f11bebed0a13a3f4bff3b",
6
+ "Manifest.json": "1254ad9d54bbd2c0bbb6a7a407e20b138a159b3ccc6e8e622e9490bf0463c90c"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 128,
12
+ "max_options": 32,
13
+ "precision": "w8enc",
14
+ "package_bytes": 534276639,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 1.2120079589658417,
34
+ "questions": 16,
35
+ "argmax_agreements": 16,
36
+ "max_probability_error": 0.03180044889450073,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 3.898020484484732,
40
+ "p95": 5.602850587456487,
41
+ "min": 3.751541953533888,
42
+ "max": 11.005959007889032
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.0001634731888771057,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 0.999755859375,
60
+ "median_ms": 4.372228984721005
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.0006970074027776718,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 1.00006103515625,
76
+ "median_ms": 3.9413750055246055
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": true,
85
+ "coreml_argmax": 1,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.007510006427764893,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 1.0001220703125,
92
+ "median_ms": 3.8934164913371205
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.006471037864685059,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 0.9998779296875,
108
+ "median_ms": 3.8704585167579353
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.03180044889450073,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 1.0,
124
+ "median_ms": 3.8540005043614656
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": true,
133
+ "coreml_argmax": 1,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 0.00015507836360484362,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 0.9999006986618042,
140
+ "median_ms": 3.916291519999504
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.009751647710800171,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 1.0,
156
+ "median_ms": 3.8382704951800406
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 0.0002784738317131996,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 1.0002403259277344,
172
+ "median_ms": 6.378958001732826
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": true,
181
+ "coreml_argmax": 0,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.00025540590286254883,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 1.0001884698867798,
188
+ "median_ms": 4.003583511803299
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.0023829713463783264,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 0.99993896484375,
204
+ "median_ms": 3.9288749685510993
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.00017899274826049805,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.000187873840332,
220
+ "median_ms": 3.880603995639831
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.0009060502052307129,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 0.9998207092285156,
236
+ "median_ms": 3.8895210018381476
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 128,
243
+ "options": 20,
244
+ "argmax_agrees": true,
245
+ "coreml_argmax": 3,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.02412015199661255,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 1.000040054321289,
252
+ "median_ms": 3.889312531100586
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.01963934302330017,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 0.999755859375,
268
+ "median_ms": 3.8546455034520477
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 128,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.0010694805532693863,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 0.9998779296875,
284
+ "median_ms": 3.8585829897783697
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 128,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.014614537358283997,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 1.0001907348632812,
300
+ "median_ms": 3.9253539871424437
301
+ }
302
+ ],
303
+ "passed": false
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 3.062132292019669,
308
+ "questions": 16,
309
+ "argmax_agreements": 11,
310
+ "max_probability_error": 0.6651095747947693,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 3.6724585224874318,
314
+ "p95": 3.8027315720682964,
315
+ "min": 3.6111659719608724,
316
+ "max": 3.8808330427855253
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": false,
327
+ "coreml_argmax": 1,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.4756666123867035,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 1.000244140625,
334
+ "median_ms": 3.727874980540946
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.17544066905975342,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 0.999755859375,
350
+ "median_ms": 3.653666499303654
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": false,
359
+ "coreml_argmax": 2,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.0822121649980545,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 1.0,
366
+ "median_ms": 3.651562496088445
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.22170449793338776,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 0.9996337890625,
382
+ "median_ms": 3.6659374891314656
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.14068716764450073,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 0.9996337890625,
398
+ "median_ms": 3.668145975098014
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": true,
407
+ "coreml_argmax": 1,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.6149341464042664,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 0.9998779296875,
414
+ "median_ms": 3.6606459761969745
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": false,
423
+ "coreml_argmax": 2,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.29022228717803955,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 0.9993896484375,
430
+ "median_ms": 3.6990624794270843
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": false,
439
+ "coreml_argmax": 1,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 0.537635862827301,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 1.0,
446
+ "median_ms": 3.6667500098701566
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": true,
455
+ "coreml_argmax": 0,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.00023287534713745117,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.9997236728668213,
462
+ "median_ms": 3.6832710029557347
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.23302960395812988,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 0.99951171875,
478
+ "median_ms": 3.757104481337592
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.5689128041267395,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.000244140625,
494
+ "median_ms": 3.7534795119427145
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": false,
503
+ "coreml_argmax": 2,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.6651095747947693,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 1.0,
510
+ "median_ms": 3.655479522421956
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 128,
517
+ "options": 20,
518
+ "argmax_agrees": true,
519
+ "coreml_argmax": 3,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.34367769956588745,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 0.9994659423828125,
526
+ "median_ms": 3.664686984848231
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.04918035864830017,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 1.0,
542
+ "median_ms": 3.648479003459215
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 128,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.43704086542129517,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 0.99951171875,
558
+ "median_ms": 3.730521013494581
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 128,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.10460180044174194,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 1.00018310546875,
574
+ "median_ms": 3.674479521578178
575
+ }
576
+ ],
577
+ "passed": false
578
+ }
579
+ },
580
+ "passed": false
581
+ }
reports/verification-multilingual-L128-w8head.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_w8head_L128_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "4cd189517c286d7bb7b302194f9ce47856c4d37ccd3c4130d9481d79d4b38a22",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "6af60903cde6e7c2f3d69b5a2c3256d4e1f8e4b01200b59f2b054f72c9ea831c",
6
+ "Manifest.json": "2bcb3450a489648654b757599cf5165372e9cff7b19c46ed68e67bb7c709c17c"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 128,
12
+ "max_options": 32,
13
+ "precision": "w8head",
14
+ "package_bytes": 629370771,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 1.3937852500239387,
34
+ "questions": 16,
35
+ "argmax_agreements": 16,
36
+ "max_probability_error": 0.0021350979804992676,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 3.881749988067895,
40
+ "p95": 4.301029801717959,
41
+ "min": 3.7419579457491636,
42
+ "max": 5.2461669547483325
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.0002027377486228943,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 1.0001220703125,
60
+ "median_ms": 3.9013955101836473
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.00014770030975341797,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 0.9998931884765625,
76
+ "median_ms": 3.899583505699411
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": true,
85
+ "coreml_argmax": 1,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.0005466341972351074,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 1.0,
92
+ "median_ms": 3.8867290131747723
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.00026895105838775635,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 1.0001220703125,
108
+ "median_ms": 3.8466039986815304
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.0021350979804992676,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 1.0001220703125,
124
+ "median_ms": 3.8555419887416065
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": true,
133
+ "coreml_argmax": 1,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 5.608797073364258e-05,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 1.000054955482483,
140
+ "median_ms": 3.81277097039856
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.0008405148983001709,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 1.0001220703125,
156
+ "median_ms": 3.865333506837487
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 4.1962601244449615e-05,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 1.0000038146972656,
172
+ "median_ms": 3.893625020282343
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": true,
181
+ "coreml_argmax": 0,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.00023287534713745117,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 0.9997756481170654,
188
+ "median_ms": 4.3462289904709905
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.00011944770812988281,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 0.9998931884765625,
204
+ "median_ms": 3.9221664774231613
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.00017899274826049805,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.0001788139343262,
220
+ "median_ms": 3.8190629857126623
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 7.051229476928711e-05,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 0.9998931884765625,
236
+ "median_ms": 3.8404999941121787
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 128,
243
+ "options": 20,
244
+ "argmax_agrees": true,
245
+ "coreml_argmax": 3,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.0006826519966125488,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 1.000295639038086,
252
+ "median_ms": 3.883999976096675
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.0011126399040222168,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 1.0,
268
+ "median_ms": 3.891729487804696
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 128,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 6.240047514438629e-05,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 0.999908447265625,
284
+ "median_ms": 3.8686875195708126
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 128,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.0010861754417419434,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 0.9999313354492188,
300
+ "median_ms": 3.864958503982052
301
+ }
302
+ ],
303
+ "passed": true
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 3.136163458984811,
308
+ "questions": 16,
309
+ "argmax_agreements": 16,
310
+ "max_probability_error": 0.012559235095977783,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 3.6686249950435013,
314
+ "p95": 3.834012206061743,
315
+ "min": 3.6094170063734055,
316
+ "max": 3.9680000045336783
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": true,
327
+ "coreml_argmax": 0,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.0004076361656188965,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 1.00018310546875,
334
+ "median_ms": 3.7582289951387793
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.00034058094024658203,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 1.0004119873046875,
350
+ "median_ms": 3.659229027107358
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": true,
359
+ "coreml_argmax": 1,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.0033596158027648926,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 1.0,
366
+ "median_ms": 3.649458463769406
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.0018558651208877563,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 1.0001220703125,
382
+ "median_ms": 3.6580205196514726
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.003983199596405029,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 1.000244140625,
398
+ "median_ms": 3.7758539838250726
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": true,
407
+ "coreml_argmax": 1,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.0004321932792663574,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 0.9995625019073486,
414
+ "median_ms": 3.6599170125555247
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": true,
423
+ "coreml_argmax": 1,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.001478821039199829,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 1.0,
430
+ "median_ms": 3.648062003776431
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": true,
439
+ "coreml_argmax": 0,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 3.820657730102539e-05,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 0.9999771118164062,
446
+ "median_ms": 3.6501249705906957
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": true,
455
+ "coreml_argmax": 0,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.00023287534713745117,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.999778151512146,
462
+ "median_ms": 3.6388119915500283
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.0004856809973716736,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 1.0003662109375,
478
+ "median_ms": 3.67008350440301
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.00017899274826049805,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.0001773834228516,
494
+ "median_ms": 3.835582989268005
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": true,
503
+ "coreml_argmax": 0,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 7.051229476928711e-05,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 0.9998779296875,
510
+ "median_ms": 3.757666505407542
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 128,
517
+ "options": 20,
518
+ "argmax_agrees": true,
519
+ "coreml_argmax": 3,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.004688441753387451,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 1.0002021789550781,
526
+ "median_ms": 3.6604584893211722
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.012559235095977783,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 1.00048828125,
542
+ "median_ms": 3.680854511912912
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 128,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.0003953631967306137,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 1.0003662109375,
558
+ "median_ms": 3.670312522444874
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 128,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.0022311657667160034,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 1.0003395080566406,
574
+ "median_ms": 3.6394580092746764
575
+ }
576
+ ],
577
+ "passed": true
578
+ }
579
+ },
580
+ "passed": true
581
+ }
reports/verification-multilingual-L256-e8.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_e8_L256_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "d16209d9b4654f87f20d9a5cbe7d8ee508c045ab3d9261d1bfc9d09392f0ade9",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "1cf24b8e0a545a1ee18a881ae1d9ff175eefc144d9da2e4674d2bd219be15a50",
6
+ "Manifest.json": "9af9f12550091738ae8b873e4de9de5dd80e948f477bdbb3ca408875a0d4562c"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 256,
12
+ "max_options": 32,
13
+ "precision": "e8",
14
+ "package_bytes": 448664434,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 6.394701374985743,
34
+ "questions": 16,
35
+ "argmax_agreements": 16,
36
+ "max_probability_error": 0.006270706653594971,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 5.778561986517161,
40
+ "p95": 6.196189246838912,
41
+ "min": 5.249582987744361,
42
+ "max": 7.637374976184219
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.0025831088423728943,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 1.00006103515625,
60
+ "median_ms": 5.771229014499113
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.0004168841987848282,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 0.9999237060546875,
76
+ "median_ms": 5.793874501250684
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": true,
85
+ "coreml_argmax": 1,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.0018501728773117065,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 0.9998779296875,
92
+ "median_ms": 5.76785349403508
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.0011222213506698608,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 1.0,
108
+ "median_ms": 5.752563010901213
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.006270706653594971,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 0.9998779296875,
124
+ "median_ms": 5.8532295224722475
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": true,
133
+ "coreml_argmax": 1,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 5.608797073364258e-05,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 1.0000249147415161,
140
+ "median_ms": 5.717207997804508
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.001695007085800171,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 1.0001220703125,
156
+ "median_ms": 5.461541499244049
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 0.00012970063835382462,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 1.000091552734375,
172
+ "median_ms": 5.813874973682687
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": true,
181
+ "coreml_argmax": 0,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.00025540590286254883,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 1.0002155303955078,
188
+ "median_ms": 5.779874976724386
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.0003025755286216736,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 1.00018310546875,
204
+ "median_ms": 5.733354017138481
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.00017899274826049805,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.0001797676086426,
220
+ "median_ms": 5.7553540100343525
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.0004576537758111954,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 1.0001220703125,
236
+ "median_ms": 5.898542003706098
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 157,
243
+ "options": 20,
244
+ "argmax_agrees": true,
245
+ "coreml_argmax": 3,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.0022539570927619934,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 1.00006103515625,
252
+ "median_ms": 5.863646016223356
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.004014343023300171,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 0.999755859375,
268
+ "median_ms": 5.812771007185802
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 174,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.0003440380096435547,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 0.9998931884765625,
284
+ "median_ms": 5.861729005118832
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 158,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.0014263391494750977,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 0.9998779296875,
300
+ "median_ms": 5.755438032792881
301
+ }
302
+ ],
303
+ "passed": true
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 8.776065583981108,
308
+ "questions": 16,
309
+ "argmax_agreements": 16,
310
+ "max_probability_error": 0.013535827398300171,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 9.623270976589993,
314
+ "p95": 9.869244421133772,
315
+ "min": 9.491292003076524,
316
+ "max": 12.103457993362099
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": true,
327
+ "coreml_argmax": 0,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.0022779330611228943,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 1.000244140625,
334
+ "median_ms": 9.617979492759332
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.0005542133003473282,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 0.999786376953125,
350
+ "median_ms": 9.594645991455764
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": true,
359
+ "coreml_argmax": 1,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.0025046318769454956,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 0.9996337890625,
366
+ "median_ms": 9.62677100324072
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.0018546432256698608,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 1.0001220703125,
382
+ "median_ms": 9.69827052904293
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.006285488605499268,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 0.99951171875,
398
+ "median_ms": 9.767146519152448
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": true,
407
+ "coreml_argmax": 1,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.0004321932792663574,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 0.9995424151420593,
414
+ "median_ms": 9.632125002099201
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": true,
423
+ "coreml_argmax": 1,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.0009625852108001709,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 0.9998779296875,
430
+ "median_ms": 9.647416474763304
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": true,
439
+ "coreml_argmax": 0,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 9.155366569757462e-05,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 1.0000534057617188,
446
+ "median_ms": 9.634103975258768
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": true,
455
+ "coreml_argmax": 0,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.00023287534713745117,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.9997386932373047,
462
+ "median_ms": 9.819416998652741
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.0005619749426841736,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 1.0004425048828125,
478
+ "median_ms": 9.638833027565852
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.00017899274826049805,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.0001797676086426,
494
+ "median_ms": 9.553520503686741
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": true,
503
+ "coreml_argmax": 0,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.0009060502052307129,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 1.0005512237548828,
510
+ "median_ms": 9.560791979311034
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 157,
517
+ "options": 20,
518
+ "argmax_agrees": true,
519
+ "coreml_argmax": 3,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.0027727559208869934,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 1.00018310546875,
526
+ "median_ms": 9.57918749190867
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.013535827398300171,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 0.99951171875,
542
+ "median_ms": 9.776624996447936
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 174,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.0003440380096435547,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 0.999847412109375,
558
+ "median_ms": 9.59639551001601
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 158,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.0009380578994750977,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 0.9996337890625,
574
+ "median_ms": 9.579125500749797
575
+ }
576
+ ],
577
+ "passed": true
578
+ }
579
+ },
580
+ "passed": true
581
+ }
reports/verification-multilingual-L512-e8.json ADDED
@@ -0,0 +1,581 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "package": "laya_multilingual_e8_L512_options32.mlpackage",
3
+ "package_files_sha256": {
4
+ "Data/com.apple.CoreML/model.mlmodel": "908a392aa7d3382b5c1c178e812a9fcfe0e741c240a9a10ce96458ef047c2c60",
5
+ "Data/com.apple.CoreML/weights/weight.bin": "70c299c32d118ab1e2549e9c0088d712bfe2e936ad37ab8f9b7366f277afed4b",
6
+ "Manifest.json": "a2cf370109daa1108d2880de4c8686191fa76926643f8f56cacafae7d0a246b9"
7
+ },
8
+ "source_repo": "convaiinnovations/laya",
9
+ "source_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982",
10
+ "variant": "multilingual",
11
+ "length": 512,
12
+ "max_options": 32,
13
+ "precision": "e8",
14
+ "package_bytes": 449516404,
15
+ "environment": {
16
+ "chip": "Apple M5 Pro",
17
+ "os": "27.0",
18
+ "python": "3.12.13",
19
+ "torch": "2.7.0",
20
+ "coremltools": "9.0"
21
+ },
22
+ "gates": {
23
+ "argmax_agreement": "100%",
24
+ "max_probability_error": 0.02,
25
+ "max_action_probability_error": 0.02,
26
+ "finite": true,
27
+ "padding_masked": true
28
+ },
29
+ "cases_sha256": "4de4e34309c421913524209b62bb7c8a0a232928bd228ec12e884347ce65eeed",
30
+ "runs": {
31
+ "ALL": {
32
+ "compute_units": "ALL",
33
+ "load_seconds": 5.690649084048346,
34
+ "questions": 16,
35
+ "argmax_agreements": 16,
36
+ "max_probability_error": 0.006270706653594971,
37
+ "max_action_probability_error": 0.0,
38
+ "latency_ms": {
39
+ "p50": 9.148353972705081,
40
+ "p95": 9.470572863938287,
41
+ "min": 8.71966703562066,
42
+ "max": 13.645083992742002
43
+ },
44
+ "skipped": [],
45
+ "rows": [
46
+ {
47
+ "case": "tetris_clean",
48
+ "question": "clean",
49
+ "type": "noul",
50
+ "tokens": 80,
51
+ "options": 2,
52
+ "argmax_agrees": true,
53
+ "coreml_argmax": 0,
54
+ "reference_argmax": 0,
55
+ "max_probability_error": 0.0025831088423728943,
56
+ "max_action_probability_error": 0.0,
57
+ "finite": true,
58
+ "padding_masked": true,
59
+ "probability_sum": 1.00006103515625,
60
+ "median_ms": 9.161458496237174
61
+ },
62
+ {
63
+ "case": "tetris_flat",
64
+ "question": "clean",
65
+ "type": "noul",
66
+ "tokens": 84,
67
+ "options": 2,
68
+ "argmax_agrees": true,
69
+ "coreml_argmax": 1,
70
+ "reference_argmax": 1,
71
+ "max_probability_error": 0.0004168841987848282,
72
+ "max_action_probability_error": 0.0,
73
+ "finite": true,
74
+ "padding_masked": true,
75
+ "probability_sum": 0.9999237060546875,
76
+ "median_ms": 9.129562502494082
77
+ },
78
+ {
79
+ "case": "tetris_lane",
80
+ "question": "lane",
81
+ "type": "choice",
82
+ "tokens": 62,
83
+ "options": 4,
84
+ "argmax_agrees": true,
85
+ "coreml_argmax": 1,
86
+ "reference_argmax": 1,
87
+ "max_probability_error": 0.0018501728773117065,
88
+ "max_action_probability_error": 0.0,
89
+ "finite": true,
90
+ "padding_masked": true,
91
+ "probability_sum": 0.9998779296875,
92
+ "median_ms": 9.263979503884912
93
+ },
94
+ {
95
+ "case": "tetris_lane",
96
+ "question": "risk",
97
+ "type": "score",
98
+ "tokens": 64,
99
+ "options": 3,
100
+ "argmax_agrees": true,
101
+ "coreml_argmax": 1,
102
+ "reference_argmax": 1,
103
+ "max_probability_error": 0.0011222213506698608,
104
+ "max_action_probability_error": 0.0,
105
+ "finite": true,
106
+ "padding_masked": true,
107
+ "probability_sum": 1.0,
108
+ "median_ms": 9.095000015804544
109
+ },
110
+ {
111
+ "case": "form_field",
112
+ "question": "action",
113
+ "type": "choice",
114
+ "tokens": 104,
115
+ "options": 3,
116
+ "argmax_agrees": true,
117
+ "coreml_argmax": 0,
118
+ "reference_argmax": 0,
119
+ "max_probability_error": 0.006270706653594971,
120
+ "max_action_probability_error": 0.0,
121
+ "finite": true,
122
+ "padding_masked": true,
123
+ "probability_sum": 0.9998779296875,
124
+ "median_ms": 9.031604480696842
125
+ },
126
+ {
127
+ "case": "support_triage",
128
+ "question": "intent",
129
+ "type": "choice",
130
+ "tokens": 62,
131
+ "options": 4,
132
+ "argmax_agrees": true,
133
+ "coreml_argmax": 1,
134
+ "reference_argmax": 1,
135
+ "max_probability_error": 5.608797073364258e-05,
136
+ "max_action_probability_error": 0.0,
137
+ "finite": true,
138
+ "padding_masked": true,
139
+ "probability_sum": 1.0000249147415161,
140
+ "median_ms": 9.217895974870771
141
+ },
142
+ {
143
+ "case": "support_triage",
144
+ "question": "frustration",
145
+ "type": "score",
146
+ "tokens": 69,
147
+ "options": 3,
148
+ "argmax_agrees": true,
149
+ "coreml_argmax": 1,
150
+ "reference_argmax": 1,
151
+ "max_probability_error": 0.001695007085800171,
152
+ "max_action_probability_error": 0.0,
153
+ "finite": true,
154
+ "padding_masked": true,
155
+ "probability_sum": 1.0001220703125,
156
+ "median_ms": 9.165416500763968
157
+ },
158
+ {
159
+ "case": "support_triage",
160
+ "question": "churn",
161
+ "type": "noul",
162
+ "tokens": 71,
163
+ "options": 2,
164
+ "argmax_agrees": true,
165
+ "coreml_argmax": 0,
166
+ "reference_argmax": 0,
167
+ "max_probability_error": 0.00012970063835382462,
168
+ "max_action_probability_error": 0.0,
169
+ "finite": true,
170
+ "padding_masked": true,
171
+ "probability_sum": 1.000091552734375,
172
+ "median_ms": 9.23962498200126
173
+ },
174
+ {
175
+ "case": "zh",
176
+ "question": "intent",
177
+ "type": "choice",
178
+ "tokens": 37,
179
+ "options": 3,
180
+ "argmax_agrees": true,
181
+ "coreml_argmax": 0,
182
+ "reference_argmax": 0,
183
+ "max_probability_error": 0.00025540590286254883,
184
+ "max_action_probability_error": 0.0,
185
+ "finite": true,
186
+ "padding_masked": true,
187
+ "probability_sum": 1.0002155303955078,
188
+ "median_ms": 8.99200001731515
189
+ },
190
+ {
191
+ "case": "zh",
192
+ "question": "urgent",
193
+ "type": "noul",
194
+ "tokens": 48,
195
+ "options": 2,
196
+ "argmax_agrees": true,
197
+ "coreml_argmax": 1,
198
+ "reference_argmax": 1,
199
+ "max_probability_error": 0.0003025755286216736,
200
+ "max_action_probability_error": 0.0,
201
+ "finite": true,
202
+ "padding_masked": true,
203
+ "probability_sum": 1.00018310546875,
204
+ "median_ms": 9.181979519780725
205
+ },
206
+ {
207
+ "case": "ja",
208
+ "question": "intent",
209
+ "type": "choice",
210
+ "tokens": 45,
211
+ "options": 3,
212
+ "argmax_agrees": true,
213
+ "coreml_argmax": 0,
214
+ "reference_argmax": 0,
215
+ "max_probability_error": 0.00017899274826049805,
216
+ "max_action_probability_error": 0.0,
217
+ "finite": true,
218
+ "padding_masked": true,
219
+ "probability_sum": 1.0001797676086426,
220
+ "median_ms": 8.985167020000517
221
+ },
222
+ {
223
+ "case": "de",
224
+ "question": "intent",
225
+ "type": "choice",
226
+ "tokens": 44,
227
+ "options": 3,
228
+ "argmax_agrees": true,
229
+ "coreml_argmax": 0,
230
+ "reference_argmax": 0,
231
+ "max_probability_error": 0.0004576537758111954,
232
+ "max_action_probability_error": 0.0,
233
+ "finite": true,
234
+ "padding_masked": true,
235
+ "probability_sum": 1.0001220703125,
236
+ "median_ms": 9.146521013462916
237
+ },
238
+ {
239
+ "case": "many_options",
240
+ "question": "lane",
241
+ "type": "choice",
242
+ "tokens": 157,
243
+ "options": 20,
244
+ "argmax_agrees": true,
245
+ "coreml_argmax": 3,
246
+ "reference_argmax": 3,
247
+ "max_probability_error": 0.0022539570927619934,
248
+ "max_action_probability_error": 0.0,
249
+ "finite": true,
250
+ "padding_masked": true,
251
+ "probability_sum": 1.00006103515625,
252
+ "median_ms": 9.232708514900878
253
+ },
254
+ {
255
+ "case": "guard",
256
+ "question": "injection",
257
+ "type": "noul",
258
+ "tokens": 48,
259
+ "options": 2,
260
+ "argmax_agrees": true,
261
+ "coreml_argmax": 1,
262
+ "reference_argmax": 1,
263
+ "max_probability_error": 0.004014343023300171,
264
+ "max_action_probability_error": 0.0,
265
+ "finite": true,
266
+ "padding_masked": true,
267
+ "probability_sum": 0.999755859375,
268
+ "median_ms": 8.971625502454117
269
+ },
270
+ {
271
+ "case": "long_state",
272
+ "question": "ready",
273
+ "type": "noul",
274
+ "tokens": 174,
275
+ "options": 2,
276
+ "argmax_agrees": true,
277
+ "coreml_argmax": 1,
278
+ "reference_argmax": 1,
279
+ "max_probability_error": 0.0003440380096435547,
280
+ "max_action_probability_error": 0.0,
281
+ "finite": true,
282
+ "padding_masked": true,
283
+ "probability_sum": 0.9998931884765625,
284
+ "median_ms": 9.061104501597583
285
+ },
286
+ {
287
+ "case": "long_state",
288
+ "question": "owner",
289
+ "type": "choice",
290
+ "tokens": 158,
291
+ "options": 3,
292
+ "argmax_agrees": true,
293
+ "coreml_argmax": 2,
294
+ "reference_argmax": 2,
295
+ "max_probability_error": 0.0014263391494750977,
296
+ "max_action_probability_error": 0.0,
297
+ "finite": true,
298
+ "padding_masked": true,
299
+ "probability_sum": 0.9998779296875,
300
+ "median_ms": 9.162145521258935
301
+ }
302
+ ],
303
+ "passed": true
304
+ },
305
+ "CPU_AND_NE": {
306
+ "compute_units": "CPU_AND_NE",
307
+ "load_seconds": 10.31118995900033,
308
+ "questions": 16,
309
+ "argmax_agreements": 16,
310
+ "max_probability_error": 0.013535827398300171,
311
+ "max_action_probability_error": 0.0,
312
+ "latency_ms": {
313
+ "p50": 27.725584019208327,
314
+ "p95": 28.007108659949154,
315
+ "min": 27.269584010355175,
316
+ "max": 32.14670799206942
317
+ },
318
+ "skipped": [],
319
+ "rows": [
320
+ {
321
+ "case": "tetris_clean",
322
+ "question": "clean",
323
+ "type": "noul",
324
+ "tokens": 80,
325
+ "options": 2,
326
+ "argmax_agrees": true,
327
+ "coreml_argmax": 0,
328
+ "reference_argmax": 0,
329
+ "max_probability_error": 0.0022779330611228943,
330
+ "max_action_probability_error": 0.0,
331
+ "finite": true,
332
+ "padding_masked": true,
333
+ "probability_sum": 1.000244140625,
334
+ "median_ms": 27.705749991582707
335
+ },
336
+ {
337
+ "case": "tetris_flat",
338
+ "question": "clean",
339
+ "type": "noul",
340
+ "tokens": 84,
341
+ "options": 2,
342
+ "argmax_agrees": true,
343
+ "coreml_argmax": 1,
344
+ "reference_argmax": 1,
345
+ "max_probability_error": 0.0005542133003473282,
346
+ "max_action_probability_error": 0.0,
347
+ "finite": true,
348
+ "padding_masked": true,
349
+ "probability_sum": 0.999786376953125,
350
+ "median_ms": 27.664041990647092
351
+ },
352
+ {
353
+ "case": "tetris_lane",
354
+ "question": "lane",
355
+ "type": "choice",
356
+ "tokens": 62,
357
+ "options": 4,
358
+ "argmax_agrees": true,
359
+ "coreml_argmax": 1,
360
+ "reference_argmax": 1,
361
+ "max_probability_error": 0.0025046318769454956,
362
+ "max_action_probability_error": 0.0,
363
+ "finite": true,
364
+ "padding_masked": true,
365
+ "probability_sum": 0.9996337890625,
366
+ "median_ms": 27.79045849456452
367
+ },
368
+ {
369
+ "case": "tetris_lane",
370
+ "question": "risk",
371
+ "type": "score",
372
+ "tokens": 64,
373
+ "options": 3,
374
+ "argmax_agrees": true,
375
+ "coreml_argmax": 1,
376
+ "reference_argmax": 1,
377
+ "max_probability_error": 0.0018546432256698608,
378
+ "max_action_probability_error": 0.0,
379
+ "finite": true,
380
+ "padding_masked": true,
381
+ "probability_sum": 1.0001220703125,
382
+ "median_ms": 27.8251460113097
383
+ },
384
+ {
385
+ "case": "form_field",
386
+ "question": "action",
387
+ "type": "choice",
388
+ "tokens": 104,
389
+ "options": 3,
390
+ "argmax_agrees": true,
391
+ "coreml_argmax": 0,
392
+ "reference_argmax": 0,
393
+ "max_probability_error": 0.006285488605499268,
394
+ "max_action_probability_error": 0.0,
395
+ "finite": true,
396
+ "padding_masked": true,
397
+ "probability_sum": 0.99951171875,
398
+ "median_ms": 27.81237449380569
399
+ },
400
+ {
401
+ "case": "support_triage",
402
+ "question": "intent",
403
+ "type": "choice",
404
+ "tokens": 62,
405
+ "options": 4,
406
+ "argmax_agrees": true,
407
+ "coreml_argmax": 1,
408
+ "reference_argmax": 1,
409
+ "max_probability_error": 0.0004321932792663574,
410
+ "max_action_probability_error": 0.0,
411
+ "finite": true,
412
+ "padding_masked": true,
413
+ "probability_sum": 0.9995424151420593,
414
+ "median_ms": 27.647562499623746
415
+ },
416
+ {
417
+ "case": "support_triage",
418
+ "question": "frustration",
419
+ "type": "score",
420
+ "tokens": 69,
421
+ "options": 3,
422
+ "argmax_agrees": true,
423
+ "coreml_argmax": 1,
424
+ "reference_argmax": 1,
425
+ "max_probability_error": 0.0009625852108001709,
426
+ "max_action_probability_error": 0.0,
427
+ "finite": true,
428
+ "padding_masked": true,
429
+ "probability_sum": 0.9998779296875,
430
+ "median_ms": 27.70072899875231
431
+ },
432
+ {
433
+ "case": "support_triage",
434
+ "question": "churn",
435
+ "type": "noul",
436
+ "tokens": 71,
437
+ "options": 2,
438
+ "argmax_agrees": true,
439
+ "coreml_argmax": 0,
440
+ "reference_argmax": 0,
441
+ "max_probability_error": 9.155366569757462e-05,
442
+ "max_action_probability_error": 0.0,
443
+ "finite": true,
444
+ "padding_masked": true,
445
+ "probability_sum": 1.0000534057617188,
446
+ "median_ms": 27.73627103306353
447
+ },
448
+ {
449
+ "case": "zh",
450
+ "question": "intent",
451
+ "type": "choice",
452
+ "tokens": 37,
453
+ "options": 3,
454
+ "argmax_agrees": true,
455
+ "coreml_argmax": 0,
456
+ "reference_argmax": 0,
457
+ "max_probability_error": 0.00023287534713745117,
458
+ "max_action_probability_error": 0.0,
459
+ "finite": true,
460
+ "padding_masked": true,
461
+ "probability_sum": 0.9997386932373047,
462
+ "median_ms": 27.639124979032204
463
+ },
464
+ {
465
+ "case": "zh",
466
+ "question": "urgent",
467
+ "type": "noul",
468
+ "tokens": 48,
469
+ "options": 2,
470
+ "argmax_agrees": true,
471
+ "coreml_argmax": 1,
472
+ "reference_argmax": 1,
473
+ "max_probability_error": 0.0005619749426841736,
474
+ "max_action_probability_error": 0.0,
475
+ "finite": true,
476
+ "padding_masked": true,
477
+ "probability_sum": 1.0004425048828125,
478
+ "median_ms": 27.770020999014378
479
+ },
480
+ {
481
+ "case": "ja",
482
+ "question": "intent",
483
+ "type": "choice",
484
+ "tokens": 45,
485
+ "options": 3,
486
+ "argmax_agrees": true,
487
+ "coreml_argmax": 0,
488
+ "reference_argmax": 0,
489
+ "max_probability_error": 0.00017899274826049805,
490
+ "max_action_probability_error": 0.0,
491
+ "finite": true,
492
+ "padding_masked": true,
493
+ "probability_sum": 1.0001797676086426,
494
+ "median_ms": 27.599229477345943
495
+ },
496
+ {
497
+ "case": "de",
498
+ "question": "intent",
499
+ "type": "choice",
500
+ "tokens": 44,
501
+ "options": 3,
502
+ "argmax_agrees": true,
503
+ "coreml_argmax": 0,
504
+ "reference_argmax": 0,
505
+ "max_probability_error": 0.0009060502052307129,
506
+ "max_action_probability_error": 0.0,
507
+ "finite": true,
508
+ "padding_masked": true,
509
+ "probability_sum": 1.0005512237548828,
510
+ "median_ms": 27.759166987380013
511
+ },
512
+ {
513
+ "case": "many_options",
514
+ "question": "lane",
515
+ "type": "choice",
516
+ "tokens": 157,
517
+ "options": 20,
518
+ "argmax_agrees": true,
519
+ "coreml_argmax": 3,
520
+ "reference_argmax": 3,
521
+ "max_probability_error": 0.0027727559208869934,
522
+ "max_action_probability_error": 0.0,
523
+ "finite": true,
524
+ "padding_masked": true,
525
+ "probability_sum": 1.00018310546875,
526
+ "median_ms": 27.80645849998109
527
+ },
528
+ {
529
+ "case": "guard",
530
+ "question": "injection",
531
+ "type": "noul",
532
+ "tokens": 48,
533
+ "options": 2,
534
+ "argmax_agrees": true,
535
+ "coreml_argmax": 1,
536
+ "reference_argmax": 1,
537
+ "max_probability_error": 0.013535827398300171,
538
+ "max_action_probability_error": 0.0,
539
+ "finite": true,
540
+ "padding_masked": true,
541
+ "probability_sum": 0.99951171875,
542
+ "median_ms": 27.632729499600828
543
+ },
544
+ {
545
+ "case": "long_state",
546
+ "question": "ready",
547
+ "type": "noul",
548
+ "tokens": 174,
549
+ "options": 2,
550
+ "argmax_agrees": true,
551
+ "coreml_argmax": 1,
552
+ "reference_argmax": 1,
553
+ "max_probability_error": 0.0003440380096435547,
554
+ "max_action_probability_error": 0.0,
555
+ "finite": true,
556
+ "padding_masked": true,
557
+ "probability_sum": 0.999847412109375,
558
+ "median_ms": 27.62754147988744
559
+ },
560
+ {
561
+ "case": "long_state",
562
+ "question": "owner",
563
+ "type": "choice",
564
+ "tokens": 158,
565
+ "options": 3,
566
+ "argmax_agrees": true,
567
+ "coreml_argmax": 2,
568
+ "reference_argmax": 2,
569
+ "max_probability_error": 0.0009380578994750977,
570
+ "max_action_probability_error": 0.0,
571
+ "finite": true,
572
+ "padding_masked": true,
573
+ "probability_sum": 0.9996337890625,
574
+ "median_ms": 27.861104463227093
575
+ }
576
+ ],
577
+ "passed": true
578
+ }
579
+ },
580
+ "passed": true
581
+ }