arach commited on
Commit
4c4807c
·
verified ·
1 Parent(s): 5f2705b

Upload local-intelligence eval results for 69d27d17074a09ff9be109c6

Browse files
eval/local_intelligence/results/jobs/69d27d17074a09ff9be109c6/google-gemma-4-e4b-it.json ADDED
@@ -0,0 +1,1083 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "summary": {
3
+ "provider": "hf-transformers",
4
+ "model": "google/gemma-4-E4B-it",
5
+ "cards": 24,
6
+ "passed": 13,
7
+ "pass_rate": 0.5417,
8
+ "average_score": 0.6181,
9
+ "median_latency_ms": 7511.01,
10
+ "exact_parse_rate": 0.0,
11
+ "normalized_parse_rate": 0.9583,
12
+ "parse_failure_rate": 0.0417
13
+ },
14
+ "rows": [
15
+ {
16
+ "card_id": "memo-auto-title",
17
+ "title": "Memo Auto-Title (<1s)",
18
+ "tier": "ship-soon",
19
+ "provider": "hf-transformers",
20
+ "model": "google/gemma-4-E4B-it",
21
+ "latency_ms": 2328.87,
22
+ "raw_text": "```json\n{\n \"title\": \"Book Dentist & Email Insurance Form\"\n}\n```<turn|>",
23
+ "grade": {
24
+ "passed": true,
25
+ "score": 1.0,
26
+ "parse_error": null,
27
+ "parse_mode": "normalized",
28
+ "format_recovered": true,
29
+ "failure_kind": null,
30
+ "assertions": [
31
+ {
32
+ "assertion": "title is not \"Voice Memo\"",
33
+ "passed": true,
34
+ "details": "title='Book Dentist & Email Insurance Form'"
35
+ },
36
+ {
37
+ "assertion": "title length <= 48",
38
+ "passed": true,
39
+ "details": "len(title)=35"
40
+ },
41
+ {
42
+ "assertion": "title contains either \"dentist\" or \"insurance\"",
43
+ "passed": true,
44
+ "details": "title='book dentist & email insurance form'"
45
+ }
46
+ ],
47
+ "parsed": {
48
+ "title": "Book Dentist & Email Insurance Form"
49
+ }
50
+ }
51
+ },
52
+ {
53
+ "card_id": "memo-type-detection",
54
+ "title": "Memo Type Detection",
55
+ "tier": "ship-soon",
56
+ "provider": "hf-transformers",
57
+ "model": "google/gemma-4-E4B-it",
58
+ "latency_ms": 1992.77,
59
+ "raw_text": "```json\n{\n \"primaryType\": \"journal\",\n \"needsReview\": false\n}\n```<turn|>",
60
+ "grade": {
61
+ "passed": false,
62
+ "score": 0.6667,
63
+ "parse_error": null,
64
+ "parse_mode": "normalized",
65
+ "format_recovered": true,
66
+ "failure_kind": "schema-or-task",
67
+ "assertions": [
68
+ {
69
+ "assertion": "primaryType is either \"journal\" or \"idea\"",
70
+ "passed": true,
71
+ "details": "primaryType='journal'"
72
+ },
73
+ {
74
+ "assertion": "confidence < 0.8",
75
+ "passed": false,
76
+ "details": "confidence=None"
77
+ },
78
+ {
79
+ "assertion": "needsReview can be true when uncertainty is high",
80
+ "passed": true,
81
+ "details": "needsReview=False"
82
+ }
83
+ ],
84
+ "parsed": {
85
+ "primaryType": "journal",
86
+ "needsReview": false
87
+ }
88
+ }
89
+ },
90
+ {
91
+ "card_id": "action-item-extraction",
92
+ "title": "Action Item Extraction With Confidence",
93
+ "tier": "ship-soon",
94
+ "provider": "hf-transformers",
95
+ "model": "google/gemma-4-E4B-it",
96
+ "latency_ms": 5662.29,
97
+ "raw_text": "```json\n[\n {\n \"text\": \"clean up the old docs folder sometime\",\n \"owner\": \"Arach\",\n \"dueDateISO\": null,\n \"confidence\": 0.8,\n \"evidence\": \"Maybe clean up the old docs folder sometime.\"\n }\n]\n```<turn|>",
98
+ "grade": {
99
+ "passed": false,
100
+ "score": 0.6667,
101
+ "parse_error": null,
102
+ "parse_mode": "normalized",
103
+ "format_recovered": true,
104
+ "failure_kind": "schema-or-task",
105
+ "assertions": [
106
+ {
107
+ "assertion": "items length >= 1",
108
+ "passed": true,
109
+ "details": "items=1"
110
+ },
111
+ {
112
+ "assertion": "item confidence < 0.75 for vague tasks",
113
+ "passed": false,
114
+ "details": "items=[{'text': 'clean up the old docs folder sometime', 'owner': 'Arach', 'dueDateISO': None, 'confidence': 0.8, 'evidence': 'Maybe clean up the old docs folder sometime.'}]"
115
+ },
116
+ {
117
+ "assertion": "evidence field is always present",
118
+ "passed": true,
119
+ "details": "items=[{'text': 'clean up the old docs folder sometime', 'owner': 'Arach', 'dueDateISO': None, 'confidence': 0.8, 'evidence': 'Maybe clean up the old docs folder sometime.'}]"
120
+ }
121
+ ],
122
+ "parsed": {
123
+ "items": [
124
+ {
125
+ "text": "clean up the old docs folder sometime",
126
+ "owner": "Arach",
127
+ "dueDateISO": null,
128
+ "confidence": 0.8,
129
+ "evidence": "Maybe clean up the old docs folder sometime."
130
+ }
131
+ ]
132
+ }
133
+ }
134
+ },
135
+ {
136
+ "card_id": "transcript-cleanup-presets",
137
+ "title": "Transcript Cleanup Presets",
138
+ "tier": "ship-soon",
139
+ "provider": "hf-transformers",
140
+ "model": "google/gemma-4-E4B-it",
141
+ "latency_ms": 3419.06,
142
+ "raw_text": "```json\n{\n \"rewritten_transcript\": [\n \"Spent 4 hours on export bugs.\",\n \"Fixed 3 of 5 issues.\"\n ]\n}\n```<turn|>",
143
+ "grade": {
144
+ "passed": true,
145
+ "score": 1.0,
146
+ "parse_error": null,
147
+ "parse_mode": "normalized",
148
+ "format_recovered": true,
149
+ "failure_kind": null,
150
+ "assertions": [
151
+ {
152
+ "assertion": "output contains number 4",
153
+ "passed": true,
154
+ "details": "needle=4"
155
+ },
156
+ {
157
+ "assertion": "output contains phrase 3 of 5",
158
+ "passed": true,
159
+ "details": "needle=3 of 5"
160
+ },
161
+ {
162
+ "assertion": "filler words are not introduced",
163
+ "passed": true,
164
+ "details": "{\"rewritten_transcript\": [\"spent 4 hours on export bugs.\", \"fixed 3 of 5 issues.\"]}"
165
+ }
166
+ ],
167
+ "parsed": {
168
+ "rewritten_transcript": [
169
+ "Spent 4 hours on export bugs.",
170
+ "Fixed 3 of 5 issues."
171
+ ]
172
+ }
173
+ }
174
+ },
175
+ {
176
+ "card_id": "what-matters-summary",
177
+ "title": "Top-3 What Matters Summary",
178
+ "tier": "ship-soon",
179
+ "provider": "hf-transformers",
180
+ "model": "google/gemma-4-E4B-it",
181
+ "latency_ms": 13376.37,
182
+ "raw_text": "```json\n{\n \"summary\": [\n {\n \"point\": \"The memo explicitly lists four distinct decisions.\",\n \"evidence\": \"Decision one. Decision two. Decision three. Decision four.\",\n \"priority_score\": 95\n },\n {\n \"point\": \"The content is purely a sequential enumeration of decisions.\",\n \"evidence\": \"Decision one. Decision two. Decision three. Decision four.\",\n \"priority_score\": 80\n },\n {\n \"point\": \"No context or details are provided regarding the nature or outcome of these decisions.\",\n \"evidence\": \"Decision one. Decision two. Decision three. Decision four.\",\n \"priority_score\": 70\n }\n ]\n}\n```<turn|>",
183
+ "grade": {
184
+ "passed": true,
185
+ "score": 1.0,
186
+ "parse_error": null,
187
+ "parse_mode": "normalized",
188
+ "format_recovered": true,
189
+ "failure_kind": null,
190
+ "assertions": [
191
+ {
192
+ "assertion": "topPoints length equals 3",
193
+ "passed": true,
194
+ "details": "topPoints=3"
195
+ },
196
+ {
197
+ "assertion": "priority values are numeric",
198
+ "passed": true,
199
+ "details": "topPoints=[{'point': 'The memo explicitly lists four distinct decisions.', 'priority': 95, 'evidence': 'Decision one. Decision two. Decision three. Decision four.'}, {'point': 'The content is purely a sequential enumeration of decisions.', 'priority': 80, 'evidence': 'Decision one. Decision two. Decision three. Decision four.'}, {'point': 'No context or details are provided regarding the nature or outcome of these decisions.', 'priority': 70, 'evidence': 'Decision one. Decision two. Decision three. Decision four.'}]"
200
+ },
201
+ {
202
+ "assertion": "each point includes evidence",
203
+ "passed": true,
204
+ "details": "topPoints=[{'point': 'The memo explicitly lists four distinct decisions.', 'priority': 95, 'evidence': 'Decision one. Decision two. Decision three. Decision four.'}, {'point': 'The content is purely a sequential enumeration of decisions.', 'priority': 80, 'evidence': 'Decision one. Decision two. Decision three. Decision four.'}, {'point': 'No context or details are provided regarding the nature or outcome of these decisions.', 'priority': 70, 'evidence': 'Decision one. Decision two. Decision three. Decision four.'}]"
205
+ }
206
+ ],
207
+ "parsed": {
208
+ "summary": [
209
+ {
210
+ "point": "The memo explicitly lists four distinct decisions.",
211
+ "evidence": "Decision one. Decision two. Decision three. Decision four.",
212
+ "priority_score": 95
213
+ },
214
+ {
215
+ "point": "The content is purely a sequential enumeration of decisions.",
216
+ "evidence": "Decision one. Decision two. Decision three. Decision four.",
217
+ "priority_score": 80
218
+ },
219
+ {
220
+ "point": "No context or details are provided regarding the nature or outcome of these decisions.",
221
+ "evidence": "Decision one. Decision two. Decision three. Decision four.",
222
+ "priority_score": 70
223
+ }
224
+ ],
225
+ "topPoints": [
226
+ {
227
+ "point": "The memo explicitly lists four distinct decisions.",
228
+ "priority": 95,
229
+ "evidence": "Decision one. Decision two. Decision three. Decision four."
230
+ },
231
+ {
232
+ "point": "The content is purely a sequential enumeration of decisions.",
233
+ "priority": 80,
234
+ "evidence": "Decision one. Decision two. Decision three. Decision four."
235
+ },
236
+ {
237
+ "point": "No context or details are provided regarding the nature or outcome of these decisions.",
238
+ "priority": 70,
239
+ "evidence": "Decision one. Decision two. Decision three. Decision four."
240
+ }
241
+ ]
242
+ }
243
+ }
244
+ },
245
+ {
246
+ "card_id": "calendar-intent-detection",
247
+ "title": "Calendar Intent Detection",
248
+ "tier": "ship-soon",
249
+ "provider": "hf-transformers",
250
+ "model": "google/gemma-4-E4B-it",
251
+ "latency_ms": 5205.73,
252
+ "raw_text": "```json\n{\n \"intent\": \"call\",\n \"arguments\": {\n \"contact\": \"Sam\",\n \"time\": \"2026-04-05T14:00:00-04:00\"\n }\n}\n```<turn|>",
253
+ "grade": {
254
+ "passed": true,
255
+ "score": 1.0,
256
+ "parse_error": null,
257
+ "parse_mode": "normalized",
258
+ "format_recovered": true,
259
+ "failure_kind": null,
260
+ "assertions": [
261
+ {
262
+ "assertion": "startAtISO date is 2026-04-05",
263
+ "passed": true,
264
+ "details": "startAtISO='2026-04-05T14:00:00-04:00'"
265
+ },
266
+ {
267
+ "assertion": "timezone offset in startAtISO is -04:00",
268
+ "passed": true,
269
+ "details": "startAtISO='2026-04-05T14:00:00-04:00'"
270
+ },
271
+ {
272
+ "assertion": "event title mentions Sam",
273
+ "passed": true,
274
+ "details": "title='Call Sam'"
275
+ }
276
+ ],
277
+ "parsed": {
278
+ "intent": "call",
279
+ "arguments": {
280
+ "contact": "Sam",
281
+ "time": "2026-04-05T14:00:00-04:00"
282
+ },
283
+ "title": "Call Sam",
284
+ "startAtISO": "2026-04-05T14:00:00-04:00"
285
+ }
286
+ }
287
+ },
288
+ {
289
+ "card_id": "reminder-normalization",
290
+ "title": "Reminder Phrasing Normalization",
291
+ "tier": "ship-soon",
292
+ "provider": "hf-transformers",
293
+ "model": "google/gemma-4-E4B-it",
294
+ "latency_ms": 3766.76,
295
+ "raw_text": "```json\n{\n \"reminderText\": \"Investigate new card design.\",\n \"dueDateISO\": null,\n \"followUpQuestion\": \"When should I look into the new card design?\"\n}\n```<turn|>",
296
+ "grade": {
297
+ "passed": true,
298
+ "score": 1.0,
299
+ "parse_error": null,
300
+ "parse_mode": "normalized",
301
+ "format_recovered": true,
302
+ "failure_kind": null,
303
+ "assertions": [
304
+ {
305
+ "assertion": "reminderText starts with a verb",
306
+ "passed": true,
307
+ "details": "reminderText='Investigate new card design.'"
308
+ },
309
+ {
310
+ "assertion": "reminderText does not contain \"maybe\"",
311
+ "passed": true,
312
+ "details": "reminderText='investigate new card design.'"
313
+ },
314
+ {
315
+ "assertion": "followUpQuestion exists when due date is missing",
316
+ "passed": true,
317
+ "details": "followUpQuestion='When should I look into the new card design?'"
318
+ }
319
+ ],
320
+ "parsed": {
321
+ "reminderText": "Investigate new card design.",
322
+ "dueDateISO": null,
323
+ "followUpQuestion": "When should I look into the new card design?"
324
+ }
325
+ }
326
+ },
327
+ {
328
+ "card_id": "private-redaction-pass",
329
+ "title": "Private Redaction Pass",
330
+ "tier": "ship-soon",
331
+ "provider": "hf-transformers",
332
+ "model": "google/gemma-4-E4B-it",
333
+ "latency_ms": 11019.62,
334
+ "raw_text": "```json\n{\n \"redacted_text\": \"SSN [REDACTED_SSN] and account [REDACTED_ACCOUNT_NUMBER] should never leave device.\",\n \"redacted_entities\": [\n {\n \"entity\": \"555-44-9012\",\n \"type\": \"SSN\",\n \"confidence\": 0.99\n },\n {\n \"entity\": \"00391299\",\n \"type\": \"ACCOUNT_NUMBER\",\n \"confidence\": 0.98\n }\n ]\n}\n```<turn|>",
335
+ "grade": {
336
+ "passed": true,
337
+ "score": 1.0,
338
+ "parse_error": null,
339
+ "parse_mode": "normalized",
340
+ "format_recovered": true,
341
+ "failure_kind": null,
342
+ "assertions": [
343
+ {
344
+ "assertion": "redactedTranscript does not contain 555-44-9012",
345
+ "passed": true,
346
+ "details": "redactedTranscript='SSN [REDACTED_SSN] and account [REDACTED_ACCOUNT_NUMBER] should never leave device.'"
347
+ },
348
+ {
349
+ "assertion": "redactedTranscript does not contain 00391299",
350
+ "passed": true,
351
+ "details": "redactedTranscript='SSN [REDACTED_SSN] and account [REDACTED_ACCOUNT_NUMBER] should never leave device.'"
352
+ },
353
+ {
354
+ "assertion": "entities length >= 2",
355
+ "passed": true,
356
+ "details": "entities=[{'value': '555-44-9012', 'type': 'SSN', 'confidence': 0.99}, {'value': '00391299', 'type': 'ACCOUNT_NUMBER', 'confidence': 0.98}]"
357
+ }
358
+ ],
359
+ "parsed": {
360
+ "redacted_text": "SSN [REDACTED_SSN] and account [REDACTED_ACCOUNT_NUMBER] should never leave device.",
361
+ "redacted_entities": [
362
+ {
363
+ "entity": "555-44-9012",
364
+ "type": "SSN",
365
+ "confidence": 0.99
366
+ },
367
+ {
368
+ "entity": "00391299",
369
+ "type": "ACCOUNT_NUMBER",
370
+ "confidence": 0.98
371
+ }
372
+ ],
373
+ "redactedTranscript": "SSN [REDACTED_SSN] and account [REDACTED_ACCOUNT_NUMBER] should never leave device.",
374
+ "entities": [
375
+ {
376
+ "value": "555-44-9012",
377
+ "type": "SSN",
378
+ "confidence": 0.99
379
+ },
380
+ {
381
+ "value": "00391299",
382
+ "type": "ACCOUNT_NUMBER",
383
+ "confidence": 0.98
384
+ }
385
+ ]
386
+ }
387
+ }
388
+ },
389
+ {
390
+ "card_id": "writing-style-memory",
391
+ "title": "Per-User Writing Style Memory",
392
+ "tier": "local-first",
393
+ "provider": "hf-transformers",
394
+ "model": "google/gemma-4-E4B-it",
395
+ "latency_ms": 2073.7,
396
+ "raw_text": "```json\n{\n \"rewritten_text\": \"We should use a new onboarding flow.\"\n}\n```<turn|>",
397
+ "grade": {
398
+ "passed": true,
399
+ "score": 1.0,
400
+ "parse_error": null,
401
+ "parse_mode": "normalized",
402
+ "format_recovered": true,
403
+ "failure_kind": null,
404
+ "assertions": [
405
+ {
406
+ "assertion": "styledSummary does not contain \"utilize\"",
407
+ "passed": true,
408
+ "details": "styledSummary='we should use a new onboarding flow.'"
409
+ },
410
+ {
411
+ "assertion": "output preserves onboarding meaning",
412
+ "passed": true,
413
+ "details": "{\"rewritten_text\": \"We should use a new onboarding flow.\", \"styledSummary\": \"We should use a new onboarding flow.\"}"
414
+ }
415
+ ],
416
+ "parsed": {
417
+ "rewritten_text": "We should use a new onboarding flow.",
418
+ "styledSummary": "We should use a new onboarding flow."
419
+ }
420
+ }
421
+ },
422
+ {
423
+ "card_id": "similar-memo-recall",
424
+ "title": "Find Similar Memo (Semantic Recall)",
425
+ "tier": "local-first",
426
+ "provider": "hf-transformers",
427
+ "model": "google/gemma-4-E4B-it",
428
+ "latency_ms": 12114.38,
429
+ "raw_text": "```json\n{\n \"top_matches\": [\n {\n \"memoId\": \"a\",\n \"relevance_score\": 0.95,\n \"rationale\": \"This memo directly addresses 'cancellation policy' and 'trials,' which are the core topics mentioned in the current transcript. This represents strong decision continuity.\"\n },\n {\n \"memoId\": \"b\",\n \"relevance_score\": 0.1,\n \"rationale\": \"This memo discusses 'policy' but focuses on 'typography updates,' which is a tangential topic and lacks continuity with the transcript's focus on cancellation and trials.\"\n }\n ]\n}\n```<turn|>",
430
+ "grade": {
431
+ "passed": true,
432
+ "score": 1.0,
433
+ "parse_error": null,
434
+ "parse_mode": "normalized",
435
+ "format_recovered": true,
436
+ "failure_kind": null,
437
+ "assertions": [
438
+ {
439
+ "assertion": "memoId \"a\" ranks first",
440
+ "passed": true,
441
+ "details": "first='a'"
442
+ },
443
+ {
444
+ "assertion": "top match relevance > 0.8",
445
+ "passed": true,
446
+ "details": "relevance=0.95"
447
+ }
448
+ ],
449
+ "parsed": {
450
+ "top_matches": [
451
+ {
452
+ "memoId": "a",
453
+ "relevance_score": 0.95,
454
+ "rationale": "This memo directly addresses 'cancellation policy' and 'trials,' which are the core topics mentioned in the current transcript. This represents strong decision continuity."
455
+ },
456
+ {
457
+ "memoId": "b",
458
+ "relevance_score": 0.1,
459
+ "rationale": "This memo discusses 'policy' but focuses on 'typography updates,' which is a tangential topic and lacks continuity with the transcript's focus on cancellation and trials."
460
+ }
461
+ ],
462
+ "matches": [
463
+ {
464
+ "memoId": "a",
465
+ "relevance": 0.95,
466
+ "rationale": "This memo directly addresses 'cancellation policy' and 'trials,' which are the core topics mentioned in the current transcript. This represents strong decision continuity."
467
+ },
468
+ {
469
+ "memoId": "b",
470
+ "relevance": 0.1,
471
+ "rationale": "This memo discusses 'policy' but focuses on 'typography updates,' which is a tangential topic and lacks continuity with the transcript's focus on cancellation and trials."
472
+ }
473
+ ]
474
+ }
475
+ }
476
+ },
477
+ {
478
+ "card_id": "project-clustering",
479
+ "title": "Project Clustering From Voice History",
480
+ "tier": "local-first",
481
+ "provider": "hf-transformers",
482
+ "model": "google/gemma-4-E4B-it",
483
+ "latency_ms": 9652.59,
484
+ "raw_text": "```json\n[\n {\n \"clusterId\": \"c1\",\n \"title\": \"Sync Queue and Reconnect Issues\",\n \"memoIds\": [\n \"a1\",\n \"a2\"\n ],\n \"confidence\": 0.95\n },\n {\n \"clusterId\": \"c2\",\n \"title\": \"Personal Notes\",\n \"memoIds\": [\n \"b1\"\n ],\n \"confidence\": 0.80\n }\n]\n```<turn|>",
485
+ "grade": {
486
+ "passed": true,
487
+ "score": 1.0,
488
+ "parse_error": null,
489
+ "parse_mode": "normalized",
490
+ "format_recovered": true,
491
+ "failure_kind": null,
492
+ "assertions": [
493
+ {
494
+ "assertion": "a1 and a2 share a clusterId",
495
+ "passed": true,
496
+ "details": "mapping={'a1': 'c1', 'a2': 'c1', 'b1': 'c2'}"
497
+ },
498
+ {
499
+ "assertion": "b1 may be separate or misc",
500
+ "passed": true,
501
+ "details": "non-blocking assertion"
502
+ }
503
+ ],
504
+ "parsed": {
505
+ "clusters": [
506
+ {
507
+ "clusterId": "c1",
508
+ "title": "Sync Queue and Reconnect Issues",
509
+ "memoIds": [
510
+ "a1",
511
+ "a2"
512
+ ],
513
+ "confidence": 0.95
514
+ },
515
+ {
516
+ "clusterId": "c2",
517
+ "title": "Personal Notes",
518
+ "memoIds": [
519
+ "b1"
520
+ ],
521
+ "confidence": 0.8
522
+ }
523
+ ]
524
+ }
525
+ }
526
+ },
527
+ {
528
+ "card_id": "next-step-suggestion-engine",
529
+ "title": "Next-Step Suggestion Engine",
530
+ "tier": "local-first",
531
+ "provider": "hf-transformers",
532
+ "model": "google/gemma-4-E4B-it",
533
+ "latency_ms": 1509.16,
534
+ "raw_text": "```json\n{\n \"action\": \"extract-tasks\"\n}\n```<turn|>",
535
+ "grade": {
536
+ "passed": false,
537
+ "score": 0.5,
538
+ "parse_error": null,
539
+ "parse_mode": "normalized",
540
+ "format_recovered": true,
541
+ "failure_kind": "schema-or-task",
542
+ "assertions": [
543
+ {
544
+ "assertion": "action is either \"extract-tasks\" or \"schedule\"",
545
+ "passed": true,
546
+ "details": "action='extract-tasks'"
547
+ },
548
+ {
549
+ "assertion": "reason references explicit deadline",
550
+ "passed": false,
551
+ "details": "reason=''"
552
+ }
553
+ ],
554
+ "parsed": {
555
+ "action": "extract-tasks"
556
+ }
557
+ }
558
+ },
559
+ {
560
+ "card_id": "follow-up-question-generator",
561
+ "title": "Follow-Up Question Generator",
562
+ "tier": "local-first",
563
+ "provider": "hf-transformers",
564
+ "model": "google/gemma-4-E4B-it",
565
+ "latency_ms": 1368.78,
566
+ "raw_text": "{\"questions\": [\"What specific aspects of the onboarding process should be improved?\"]}<turn|>",
567
+ "grade": {
568
+ "passed": true,
569
+ "score": 1.0,
570
+ "parse_error": null,
571
+ "parse_mode": "normalized",
572
+ "format_recovered": true,
573
+ "failure_kind": null,
574
+ "assertions": [
575
+ {
576
+ "assertion": "questions length equals 1",
577
+ "passed": true,
578
+ "details": "questions=['What specific aspects of the onboarding process should be improved?']"
579
+ },
580
+ {
581
+ "assertion": "question ends with ?",
582
+ "passed": true,
583
+ "details": "question='What specific aspects of the onboarding process should be improved?'"
584
+ }
585
+ ],
586
+ "parsed": {
587
+ "questions": [
588
+ "What specific aspects of the onboarding process should be improved?"
589
+ ]
590
+ }
591
+ }
592
+ },
593
+ {
594
+ "card_id": "hybrid-model-router",
595
+ "title": "Hybrid Local/Cloud Router",
596
+ "tier": "local-first",
597
+ "provider": "hf-transformers",
598
+ "model": "google/gemma-4-E4B-it",
599
+ "latency_ms": 710.33,
600
+ "raw_text": "{\"error\": \"Missing request metadata\"}<turn|>",
601
+ "grade": {
602
+ "passed": false,
603
+ "score": 0.0,
604
+ "parse_error": null,
605
+ "parse_mode": "normalized",
606
+ "format_recovered": true,
607
+ "failure_kind": "schema-or-task",
608
+ "assertions": [
609
+ {
610
+ "assertion": "route equals \"cloud\"",
611
+ "passed": false,
612
+ "details": "route=None"
613
+ },
614
+ {
615
+ "assertion": "reason mentions long context",
616
+ "passed": false,
617
+ "details": "reason=''"
618
+ }
619
+ ],
620
+ "parsed": {
621
+ "error": "Missing request metadata"
622
+ }
623
+ }
624
+ },
625
+ {
626
+ "card_id": "context-packet-builder",
627
+ "title": "Context Packet Builder",
628
+ "tier": "local-first",
629
+ "provider": "hf-transformers",
630
+ "model": "google/gemma-4-E4B-it",
631
+ "latency_ms": 12906.14,
632
+ "raw_text": "```json\n{\n \"context_packet\": {\n \"priorities\": [\n \"Plan hiring strategy\",\n \"Plan product launch\",\n \"Develop documentation strategy\",\n \"Define support plan\",\n \"Determine pricing model\",\n \"Establish analytics framework\"\n ],\n \"active_tasks\": [],\n \"unresolved_questions\": [\n \"What is the scope/timeline for the hiring plan?\",\n \"What are the key milestones for the product launch?\",\n \"What are the target user segments for pricing?\",\n \"What are the initial KPIs for analytics?\"\n ],\n \"next_steps\": \"Prioritize and break down the six planning areas into actionable tasks.\"\n }\n}\n```<turn|>",
633
+ "grade": {
634
+ "passed": true,
635
+ "score": 1.0,
636
+ "parse_error": null,
637
+ "parse_mode": "normalized",
638
+ "format_recovered": true,
639
+ "failure_kind": null,
640
+ "assertions": [
641
+ {
642
+ "assertion": "tokenEstimate <= 120",
643
+ "passed": true,
644
+ "details": "tokenEstimate=39"
645
+ },
646
+ {
647
+ "assertion": "packet has at least one decision or openQuestion field",
648
+ "passed": true,
649
+ "details": "packet={'decisions': [], 'activeTasks': [], 'openQuestions': ['What is the scope/timeline for the hiring plan?', 'What are the key milestones for the product launch?', 'What are the target user segments for pricing?', 'What are the initial KPIs for analytics?'], 'relevantMemoIds': []}"
650
+ }
651
+ ],
652
+ "parsed": {
653
+ "context_packet": {
654
+ "priorities": [
655
+ "Plan hiring strategy",
656
+ "Plan product launch",
657
+ "Develop documentation strategy",
658
+ "Define support plan",
659
+ "Determine pricing model",
660
+ "Establish analytics framework"
661
+ ],
662
+ "active_tasks": [],
663
+ "unresolved_questions": [
664
+ "What is the scope/timeline for the hiring plan?",
665
+ "What are the key milestones for the product launch?",
666
+ "What are the target user segments for pricing?",
667
+ "What are the initial KPIs for analytics?"
668
+ ],
669
+ "next_steps": "Prioritize and break down the six planning areas into actionable tasks."
670
+ },
671
+ "packet": {
672
+ "decisions": [],
673
+ "activeTasks": [],
674
+ "openQuestions": [
675
+ "What is the scope/timeline for the hiring plan?",
676
+ "What are the key milestones for the product launch?",
677
+ "What are the target user segments for pricing?",
678
+ "What are the initial KPIs for analytics?"
679
+ ],
680
+ "relevantMemoIds": []
681
+ },
682
+ "tokenEstimate": 39
683
+ }
684
+ }
685
+ },
686
+ {
687
+ "card_id": "conversation-to-checklist",
688
+ "title": "Conversation To Checklist Transformer",
689
+ "tier": "local-first",
690
+ "provider": "hf-transformers",
691
+ "model": "google/gemma-4-E4B-it",
692
+ "latency_ms": 12297.05,
693
+ "raw_text": "```json\n[\n {\n \"id\": \"1\",\n \"text\": \"export data\",\n \"order\": 1,\n \"dependsOn\": null,\n \"owner\": \"Arach\"\n },\n {\n \"id\": \"2\",\n \"text\": \"verify CSV\",\n \"order\": 2,\n \"dependsOn\": \"1\",\n \"owner\": \"Arach\"\n },\n {\n \"id\": \"3\",\n \"text\": \"email finance\",\n \"order\": 3,\n \"dependsOn\": \"2\",\n \"owner\": \"Arach\"\n }\n]\n```<turn|>",
694
+ "grade": {
695
+ "passed": false,
696
+ "score": 0.0,
697
+ "parse_error": null,
698
+ "parse_mode": "normalized",
699
+ "format_recovered": true,
700
+ "failure_kind": "schema-or-task",
701
+ "assertions": [
702
+ {
703
+ "assertion": "order values are 1,2,3",
704
+ "passed": false,
705
+ "details": "orders=[]"
706
+ },
707
+ {
708
+ "assertion": "item 2 depends on item 1",
709
+ "passed": false,
710
+ "details": "dependsOn=[]"
711
+ },
712
+ {
713
+ "assertion": "item 3 depends on item 2 or item 1",
714
+ "passed": false,
715
+ "details": "dependsOn=[]"
716
+ }
717
+ ],
718
+ "parsed": [
719
+ {
720
+ "id": "1",
721
+ "text": "export data",
722
+ "order": 1,
723
+ "dependsOn": null,
724
+ "owner": "Arach"
725
+ },
726
+ {
727
+ "id": "2",
728
+ "text": "verify CSV",
729
+ "order": 2,
730
+ "dependsOn": "1",
731
+ "owner": "Arach"
732
+ },
733
+ {
734
+ "id": "3",
735
+ "text": "email finance",
736
+ "order": 3,
737
+ "dependsOn": "2",
738
+ "owner": "Arach"
739
+ }
740
+ ]
741
+ }
742
+ },
743
+ {
744
+ "card_id": "personal-knowledge-graph",
745
+ "title": "Personal Knowledge Graph From Voice",
746
+ "tier": "big-swings",
747
+ "provider": "hf-transformers",
748
+ "model": "google/gemma-4-E4B-it",
749
+ "latency_ms": 17164.2,
750
+ "raw_text": "```json\n{\n \"nodes\": [\n {\n \"id\": \"n1\",\n \"type\": \"person\",\n \"name\": \"Chris\"\n },\n {\n \"id\": \"n2\",\n \"type\": \"decision\",\n \"name\": \"Pricing discussion\"\n },\n {\n \"id\": \"n3\",\n \"type\": \"artifact\",\n \"name\": \"Pricing confirmation\"\n }\n ],\n \"edges\": [\n {\n \"type\": \"INITIATED\",\n \"sourceNodeId\": \"n2\",\n \"targetNodeId\": \"n1\",\n \"evidenceMemoId\": \"a\"\n },\n {\n \"type\": \"CONFIRMED\",\n \"sourceNodeId\": \"n1\",\n \"targetNodeId\": \"n3\",\n \"evidenceMemoId\": \"b\"\n }\n ]\n}\n```<turn|>",
751
+ "grade": {
752
+ "passed": false,
753
+ "score": 0.0,
754
+ "parse_error": null,
755
+ "parse_mode": "normalized",
756
+ "format_recovered": true,
757
+ "failure_kind": "schema-or-task",
758
+ "assertions": [
759
+ {
760
+ "assertion": "only one node labeled Chris",
761
+ "passed": false,
762
+ "details": "count=0"
763
+ },
764
+ {
765
+ "assertion": "edges reference merged Chris node id",
766
+ "passed": false,
767
+ "details": "chris_ids=set()"
768
+ }
769
+ ],
770
+ "parsed": {
771
+ "nodes": [
772
+ {
773
+ "id": "n1",
774
+ "type": "person",
775
+ "name": "Chris"
776
+ },
777
+ {
778
+ "id": "n2",
779
+ "type": "decision",
780
+ "name": "Pricing discussion"
781
+ },
782
+ {
783
+ "id": "n3",
784
+ "type": "artifact",
785
+ "name": "Pricing confirmation"
786
+ }
787
+ ],
788
+ "edges": [
789
+ {
790
+ "type": "INITIATED",
791
+ "sourceNodeId": "n2",
792
+ "targetNodeId": "n1",
793
+ "evidenceMemoId": "a"
794
+ },
795
+ {
796
+ "type": "CONFIRMED",
797
+ "sourceNodeId": "n1",
798
+ "targetNodeId": "n3",
799
+ "evidenceMemoId": "b"
800
+ }
801
+ ]
802
+ }
803
+ }
804
+ },
805
+ {
806
+ "card_id": "catch-me-up-daily-brief",
807
+ "title": "Catch Me Up Daily Brief",
808
+ "tier": "big-swings",
809
+ "provider": "hf-transformers",
810
+ "model": "google/gemma-4-E4B-it",
811
+ "latency_ms": 8661.52,
812
+ "raw_text": "```json\n{\n \"wins\": [\n \"No critical blockers identified today.\",\n \"Maintained focus on planning activities.\"\n ],\n \"risks\": [\n \"Lack of concrete progress on key deliverables.\",\n \"Budget finalization remains outstanding.\"\n ],\n \"nextActions\": [\n \"Prioritize and finalize the budget.\",\n \"Transition from planning to execution tasks.\",\n \"Schedule a follow-up to review planning outcomes.\"\n ]\n}\n```<turn|>",
813
+ "grade": {
814
+ "passed": true,
815
+ "score": 1.0,
816
+ "parse_error": null,
817
+ "parse_mode": "normalized",
818
+ "format_recovered": true,
819
+ "failure_kind": null,
820
+ "assertions": [
821
+ {
822
+ "assertion": "wins section exists",
823
+ "passed": true,
824
+ "details": "wins=['No critical blockers identified today.', 'Maintained focus on planning activities.']"
825
+ },
826
+ {
827
+ "assertion": "risks section exists",
828
+ "passed": true,
829
+ "details": "risks=['Lack of concrete progress on key deliverables.', 'Budget finalization remains outstanding.']"
830
+ },
831
+ {
832
+ "assertion": "nextActions section exists",
833
+ "passed": true,
834
+ "details": "nextActions=['Prioritize and finalize the budget.', 'Transition from planning to execution tasks.', 'Schedule a follow-up to review planning outcomes.']"
835
+ }
836
+ ],
837
+ "parsed": {
838
+ "wins": [
839
+ "No critical blockers identified today.",
840
+ "Maintained focus on planning activities."
841
+ ],
842
+ "risks": [
843
+ "Lack of concrete progress on key deliverables.",
844
+ "Budget finalization remains outstanding."
845
+ ],
846
+ "nextActions": [
847
+ "Prioritize and finalize the budget.",
848
+ "Transition from planning to execution tasks.",
849
+ "Schedule a follow-up to review planning outcomes."
850
+ ]
851
+ }
852
+ }
853
+ },
854
+ {
855
+ "card_id": "contradiction-drift-detection",
856
+ "title": "Contradiction And Drift Detection",
857
+ "tier": "big-swings",
858
+ "provider": "hf-transformers",
859
+ "model": "google/gemma-4-E4B-it",
860
+ "latency_ms": 7473.96,
861
+ "raw_text": "```json\n{\n \"conflicts\": [\n {\n \"severity\": \"high\",\n \"conflictingDecisionId\": \"d9\",\n \"explanation\": \"The new memo ('new_test_01') states to move the quarterly review to June 2, which directly contradicts the prior decision ('d9') that fixed the quarterly review date to June 20.\"\n }\n ]\n}\n```<turn|>",
862
+ "grade": {
863
+ "passed": true,
864
+ "score": 1.0,
865
+ "parse_error": null,
866
+ "parse_mode": "normalized",
867
+ "format_recovered": true,
868
+ "failure_kind": null,
869
+ "assertions": [
870
+ {
871
+ "assertion": "conflicts length >= 1",
872
+ "passed": true,
873
+ "details": "conflicts=[{'severity': 'high', 'conflictingDecisionId': 'd9', 'explanation': \"The new memo ('new_test_01') states to move the quarterly review to June 2, which directly contradicts the prior decision ('d9') that fixed the quarterly review date to June 20.\"}]"
874
+ },
875
+ {
876
+ "assertion": "first conflict severity equals high",
877
+ "passed": true,
878
+ "details": "severity='high'"
879
+ }
880
+ ],
881
+ "parsed": {
882
+ "conflicts": [
883
+ {
884
+ "severity": "high",
885
+ "conflictingDecisionId": "d9",
886
+ "explanation": "The new memo ('new_test_01') states to move the quarterly review to June 2, which directly contradicts the prior decision ('d9') that fixed the quarterly review date to June 20."
887
+ }
888
+ ]
889
+ }
890
+ }
891
+ },
892
+ {
893
+ "card_id": "momentum-scoring",
894
+ "title": "Momentum Scoring",
895
+ "tier": "big-swings",
896
+ "provider": "hf-transformers",
897
+ "model": "google/gemma-4-E4B-it",
898
+ "latency_ms": 1322.1,
899
+ "raw_text": "```json\n{\n \"score\": 20\n}\n```<turn|>",
900
+ "grade": {
901
+ "passed": false,
902
+ "score": 0.0,
903
+ "parse_error": null,
904
+ "parse_mode": "normalized",
905
+ "format_recovered": true,
906
+ "failure_kind": "schema-or-task",
907
+ "assertions": [
908
+ {
909
+ "assertion": "momentumScore < 50",
910
+ "passed": false,
911
+ "details": "momentumScore=None"
912
+ },
913
+ {
914
+ "assertion": "unresolvedHotspots includes \"Fix onboarding bug\"",
915
+ "passed": false,
916
+ "details": "unresolvedHotspots=[]"
917
+ }
918
+ ],
919
+ "parsed": {
920
+ "score": 20
921
+ }
922
+ }
923
+ },
924
+ {
925
+ "card_id": "local-agent-loop",
926
+ "title": "Local Agent Loop For Memo Processing",
927
+ "tier": "big-swings",
928
+ "provider": "hf-transformers",
929
+ "model": "google/gemma-4-E4B-it",
930
+ "latency_ms": 19847.86,
931
+ "raw_text": "```json\n{\n \"pipeline_id\": \"memo_test_loop_01\",\n \"steps_executed\": [\n {\n \"step_name\": \"classify\",\n \"input\": \"Maybe move something around at some point.\",\n \"output\": {\n \"category\": \"General Suggestion\",\n \"confidence\": 0.95\n },\n \"status\": \"completed\"\n },\n {\n \"step_name\": \"extract\",\n \"input\": \"Maybe move something around at some point.\",\n \"output\": {\n \"keywords\": [\"move\", \"something\", \"around\"],\n \"entities\": []\n },\n \"status\": \"completed\"\n },\n {\n \"step_name\": \"summarize\",\n \"input\": \"Maybe move something around at some point.\",\n \"output\": \"A suggestion to rearrange items at some future time.\",\n \"status\": \"completed\"\n },\n {\n \"step_name\": \"route\",\n \"input\": \"A suggestion to rearrange items at some future time.\",\n \"output\": {\n \"",
932
+ "grade": {
933
+ "passed": false,
934
+ "score": 0.0,
935
+ "parse_error": "Expecting value: line 1 column 1 (char 0)",
936
+ "parse_mode": "failed",
937
+ "format_recovered": false,
938
+ "failure_kind": "parse",
939
+ "assertions": [
940
+ {
941
+ "assertion": "verify.passed can be false for vague memo",
942
+ "passed": false,
943
+ "details": "output was not valid JSON"
944
+ },
945
+ {
946
+ "assertion": "finalStatus equals \"needs-review\" when verify fails",
947
+ "passed": false,
948
+ "details": "output was not valid JSON"
949
+ }
950
+ ],
951
+ "parsed": null
952
+ }
953
+ },
954
+ {
955
+ "card_id": "meeting-live-structure",
956
+ "title": "Meeting Mode Live Rolling Structure",
957
+ "tier": "big-swings",
958
+ "provider": "hf-transformers",
959
+ "model": "google/gemma-4-E4B-it",
960
+ "latency_ms": 9053.55,
961
+ "raw_text": "```json\n{\n \"meetingId\": \"meeting_test_live_01\",\n \"state\": {\n \"agenda\": [],\n \"decisions\": [\n {\n \"text\": \"Keep beta closed for now.\",\n \"chunkIndex\": 2\n },\n {\n \"text\": \"Keep beta closed for now.\",\n \"chunkIndex\": 8\n }\n ],\n \"openQuestions\": [],\n \"actionItems\": []\n }\n}\n```<turn|>",
962
+ "grade": {
963
+ "passed": false,
964
+ "score": 0.0,
965
+ "parse_error": null,
966
+ "parse_mode": "normalized",
967
+ "format_recovered": true,
968
+ "failure_kind": "schema-or-task",
969
+ "assertions": [
970
+ {
971
+ "assertion": "updatedState.decisions length stays 1",
972
+ "passed": false,
973
+ "details": "decisions=[]"
974
+ },
975
+ {
976
+ "assertion": "existing decision remains unchanged",
977
+ "passed": false,
978
+ "details": "decisions='[]'"
979
+ }
980
+ ],
981
+ "parsed": {
982
+ "meetingId": "meeting_test_live_01",
983
+ "state": {
984
+ "agenda": [],
985
+ "decisions": [
986
+ {
987
+ "text": "Keep beta closed for now.",
988
+ "chunkIndex": 2
989
+ },
990
+ {
991
+ "text": "Keep beta closed for now.",
992
+ "chunkIndex": 8
993
+ }
994
+ ],
995
+ "openQuestions": [],
996
+ "actionItems": []
997
+ }
998
+ }
999
+ }
1000
+ },
1001
+ {
1002
+ "card_id": "intent-trend-alerts",
1003
+ "title": "Intent Trend Alerts",
1004
+ "tier": "big-swings",
1005
+ "provider": "hf-transformers",
1006
+ "model": "google/gemma-4-E4B-it",
1007
+ "latency_ms": 9805.58,
1008
+ "raw_text": "```json\n{\n \"alerts\": [\n {\n \"intent\": \"Refactor UI\",\n \"mentionCount\": 2,\n \"completionCount\": 0,\n \"alert\": true,\n \"reason\": \"Mention count (2) is below the minimum threshold (4) but completion count (0) is at the maximum threshold (0).\",\n \"suggested_intervention\": \"Review recent UI refactoring requests to understand the blockers preventing completion and prioritize a small, achievable task to build momentum.\"\n }\n ]\n}\n```<turn|>",
1009
+ "grade": {
1010
+ "passed": false,
1011
+ "score": 0.0,
1012
+ "parse_error": null,
1013
+ "parse_mode": "normalized",
1014
+ "format_recovered": true,
1015
+ "failure_kind": "schema-or-task",
1016
+ "assertions": [
1017
+ {
1018
+ "assertion": "alerts length equals 0",
1019
+ "passed": false,
1020
+ "details": "alerts=[{'intent': 'Refactor UI', 'mentionCount': 2, 'completionCount': 0, 'alert': True, 'reason': 'Mention count (2) is below the minimum threshold (4) but completion count (0) is at the maximum threshold (0).', 'suggested_intervention': 'Review recent UI refactoring requests to understand the blockers preventing completion and prioritize a small, achievable task to build momentum.'}]"
1021
+ }
1022
+ ],
1023
+ "parsed": {
1024
+ "alerts": [
1025
+ {
1026
+ "intent": "Refactor UI",
1027
+ "mentionCount": 2,
1028
+ "completionCount": 0,
1029
+ "alert": true,
1030
+ "reason": "Mention count (2) is below the minimum threshold (4) but completion count (0) is at the maximum threshold (0).",
1031
+ "suggested_intervention": "Review recent UI refactoring requests to understand the blockers preventing completion and prioritize a small, achievable task to build momentum."
1032
+ }
1033
+ ]
1034
+ }
1035
+ }
1036
+ },
1037
+ {
1038
+ "card_id": "voice-os-command-layer",
1039
+ "title": "Voice-Based Operating Layer",
1040
+ "tier": "big-swings",
1041
+ "provider": "hf-transformers",
1042
+ "model": "google/gemma-4-E4B-it",
1043
+ "latency_ms": 7548.05,
1044
+ "raw_text": "```json\n[\n {\n \"app\": \"finder\",\n \"action\": \"delete_folder\",\n \"args\": {\n \"path\": \"old exports folder\"\n },\n \"requiresConfirmation\": true\n },\n {\n \"app\": \"finder\",\n \"action\": \"empty_trash\",\n \"args\": {},\n \"requiresConfirmation\": true\n }\n]\n```<turn|>",
1045
+ "grade": {
1046
+ "passed": false,
1047
+ "score": 0.0,
1048
+ "parse_error": null,
1049
+ "parse_mode": "normalized",
1050
+ "format_recovered": true,
1051
+ "failure_kind": "schema-or-task",
1052
+ "assertions": [
1053
+ {
1054
+ "assertion": "requiresConfirmation equals true",
1055
+ "passed": false,
1056
+ "details": "requiresConfirmation=None"
1057
+ },
1058
+ {
1059
+ "assertion": "at least one step action is marked destructive",
1060
+ "passed": false,
1061
+ "details": "steps=[]"
1062
+ }
1063
+ ],
1064
+ "parsed": [
1065
+ {
1066
+ "app": "finder",
1067
+ "action": "delete_folder",
1068
+ "args": {
1069
+ "path": "old exports folder"
1070
+ },
1071
+ "requiresConfirmation": true
1072
+ },
1073
+ {
1074
+ "app": "finder",
1075
+ "action": "empty_trash",
1076
+ "args": {},
1077
+ "requiresConfirmation": true
1078
+ }
1079
+ ]
1080
+ }
1081
+ }
1082
+ ]
1083
+ }