Hugging Face's logo

Spaces:
EnvLoop
/
Pev-Leaderboard
Running

App Files Community
Pev-Leaderboard
File size: 52,075 Bytes
1c6d154
d52642e
 
 
 
177849c
 
d52642e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177849c
 
d52642e
177849c
 
 
 
 
d52642e
177849c
d52642e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177849c
d52642e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177849c
d52642e
 
 
 
177849c
d52642e
 
 
 
 
 
 
177849c
d52642e
 
 
 
 
 
 
177849c
 
d52642e
 
 
 
 
177849c
d52642e
 
 
 
 
 
 
 
 
177849c
d52642e
 
 
177849c
 
d52642e
 
177849c
 
d52642e
 
177849c
d52642e
 
 
 
 
 
 
 
 
 
 
 
 
 
177849c
 
d52642e
 
177849c
d52642e
 
 
 
 
 
 
 
 
177849c
d52642e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177849c
d52642e
 
177849c
d52642e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177849c
d52642e
 
 
177849c
d52642e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177849c
d52642e
177849c
d52642e
 
 
177849c
d52642e
 
177849c
d52642e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177849c
d52642e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177849c
d52642e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1c6d154
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Pev Leaderboard</title>
<meta name="description" content="Leaderboard for Pev-Bench: family-macro accuracy, paired comparisons, calibration and automation coverage for personal-agent decision questions.">
<meta name="color-scheme" content="light dark">
<style>
:root {
  --bg: #f7f7f5;
  --surface: #ffffff;
  --surface-2: #f0f0ed;
  --text: #1c1d1f;
  --muted: #5d6066;
  --border: #dcdcd7;
  --accent: #2a62c9;
  --accent-soft: #e8effb;
  --hl: #eef3fc;
  --hl-border: #2a62c9;
  --warn-bg: #fbf5e6;
  --warn-border: #d9b25c;
  --good: #1f7a4d;
  --code-bg: #f0f0ed;
  --figure-card: #fcfcfa;
  --focus: #2a62c9;
  --radius: 10px;
  --font: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
  --mono: ui-monospace, SFMono-Regular, Menlo, Consolas, "Liberation Mono", monospace;
}
@media (prefers-color-scheme: dark) {
  :root {
    --bg: #141517;
    --surface: #1c1d20;
    --surface-2: #25272b;
    --text: #e6e6e3;
    --muted: #a3a6ad;
    --border: #34363b;
    --accent: #7ea6f0;
    --accent-soft: #1f2a3d;
    --hl: #1d2638;
    --hl-border: #7ea6f0;
    --warn-bg: #2a2416;
    --warn-border: #8a6d2c;
    --good: #6cc196;
    --code-bg: #25272b;
    --figure-card: #fcfcfa;
    --focus: #9dbcf5;
  }
}
* { box-sizing: border-box; }
html { -webkit-text-size-adjust: 100%; }
body {
  margin: 0;
  background: var(--bg);
  color: var(--text);
  font-family: var(--font);
  font-size: 16px;
  line-height: 1.55;
  overflow-x: hidden;
}
a { color: var(--accent); }
a:focus-visible, button:focus-visible, select:focus-visible, input:focus-visible + label {
  outline: 2px solid var(--focus);
  outline-offset: 2px;
}
.wrap { max-width: 1120px; margin: 0 auto; padding: 0 16px; }
header.site { padding: 40px 0 24px; border-bottom: 1px solid var(--border); background: var(--surface); }
h1 { font-size: 2rem; line-height: 1.2; margin: 0 0 8px; letter-spacing: -0.01em; }
h2 { font-size: 1.35rem; margin: 0 0 8px; line-height: 1.3; }
h3 { font-size: 1.05rem; margin: 24px 0 8px; }
p { margin: 0 0 12px; }
.lede { font-size: 1.05rem; color: var(--muted); max-width: 820px; }
.links { display: flex; flex-wrap: wrap; gap: 8px; margin: 18px 0 12px; padding: 0; list-style: none; }
.links a {
  display: inline-block; padding: 6px 14px; border: 1px solid var(--border); border-radius: 999px;
  text-decoration: none; color: var(--text); background: var(--surface); font-size: 0.95rem;
}
.links a:hover { border-color: var(--accent); color: var(--accent); }
.note { font-size: 0.9rem; color: var(--muted); }
nav.toc { position: sticky; top: 0; z-index: 5; background: var(--bg); border-bottom: 1px solid var(--border); }
nav.toc ul { list-style: none; margin: 0 auto; padding: 8px 16px; display: flex; gap: 4px 16px; flex-wrap: wrap; max-width: 1120px; }
nav.toc a { text-decoration: none; color: var(--muted); font-size: 0.9rem; }
nav.toc a:hover { color: var(--accent); }
main section { padding: 32px 0 8px; }
main section, footer.site { scroll-margin-top: 56px; }
.card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); padding: 16px; }
.table-scroll { overflow-x: auto; -webkit-overflow-scrolling: touch; border: 1px solid var(--border); border-radius: var(--radius); background: var(--surface); margin: 12px 0; }
table { border-collapse: separate; border-spacing: 0; width: 100%; font-size: 0.92rem; }
caption { text-align: left; padding: 10px 12px; color: var(--muted); font-size: 0.88rem; caption-side: top; }
th, td { padding: 8px 12px; border-bottom: 1px solid var(--border); text-align: right; white-space: nowrap; }
th { font-weight: 600; background: var(--surface-2); }
thead th { position: relative; }
th.l, td.l { text-align: left; }
tbody tr:last-child th, tbody tr:last-child td { border-bottom: 0; }
td.num, th.num { font-variant-numeric: tabular-nums; }
.sticky { position: sticky; left: 0; z-index: 1; background: var(--surface); }
thead .sticky { background: var(--surface-2); z-index: 2; }
tbody th { font-weight: 500; text-align: left; }
tr.ours th, tr.ours td { background: var(--hl); }
tr.ours th.sticky { box-shadow: inset 3px 0 0 var(--hl-border); }
tr.ours th { font-weight: 650; }
tr.chance th, tr.chance td { color: var(--muted); font-style: italic; }
.grp { text-align: center; border-left: 1px solid var(--border); }
.bl { border-left: 1px solid var(--border); }
.tag { display: inline-block; font-size: 0.72rem; padding: 1px 6px; border-radius: 4px; border: 1px solid var(--border); color: var(--muted); font-weight: 500; margin-left: 6px; vertical-align: 1px; }
.dim { color: var(--muted); }
.controls { display: flex; flex-wrap: wrap; gap: 12px 24px; align-items: center; margin: 12px 0 4px; }
fieldset.seg { border: 0; padding: 0; margin: 0; display: inline-flex; flex-wrap: wrap; gap: 0; }
fieldset.seg legend { font-size: 0.9rem; color: var(--muted); margin-bottom: 4px; padding: 0; width: 100%; }
.seg input { position: absolute; opacity: 0; width: 1px; height: 1px; }
.seg label { padding: 6px 12px; border: 1px solid var(--border); background: var(--surface); cursor: pointer; font-size: 0.9rem; }
.seg label:first-of-type { border-radius: 8px 0 0 8px; }
.seg label:last-of-type { border-radius: 0 8px 8px 0; border-left: 0; }
.seg input:checked + label { background: var(--accent-soft); border-color: var(--accent); color: var(--text); font-weight: 600; }
.field { display: flex; flex-direction: column; gap: 4px; }
.field label { font-size: 0.9rem; color: var(--muted); }
select { font: inherit; font-size: 0.95rem; padding: 6px 10px; border-radius: 8px; border: 1px solid var(--border); background: var(--surface); color: var(--text); max-width: 100%; }
.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(210px, 1fr)); gap: 12px; margin: 16px 0; }
.stat { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); padding: 14px 16px; }
.stat .k { font-size: 0.85rem; color: var(--muted); }
.stat .v { font-size: 1.35rem; font-weight: 650; font-variant-numeric: tabular-nums; margin-top: 2px; }
.stat .s { font-size: 0.85rem; color: var(--muted); margin-top: 2px; }
.famdesc { margin: 8px 0 0; font-size: 0.95rem; }
.callout { background: var(--warn-bg); border: 1px solid var(--warn-border); border-radius: var(--radius); padding: 12px 14px; margin: 12px 0; font-size: 0.95rem; }
ul.caveats { padding-left: 20px; margin: 8px 0; }
ul.caveats li { margin-bottom: 10px; }
code { font-family: var(--mono); font-size: 0.88em; background: var(--code-bg); padding: 1px 4px; border-radius: 4px; }
.figs { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 460px), 1fr)); gap: 16px; margin-top: 12px; }
figure { margin: 0; background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); overflow: hidden; }
figure .img { background: var(--figure-card); padding: 8px; }
figure img { display: block; width: 100%; height: auto; }
figcaption { padding: 10px 14px 12px; font-size: 0.9rem; color: var(--muted); }
footer.site { margin-top: 40px; padding: 32px 0 48px; border-top: 1px solid var(--border); background: var(--surface); }
pre.bib { background: var(--code-bg); border: 1px solid var(--border); border-radius: var(--radius); padding: 12px 14px; overflow-x: auto; font-family: var(--mono); font-size: 0.85rem; line-height: 1.5; margin: 8px 0; }
.bibbar { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; }
button.copy { font: inherit; font-size: 0.9rem; padding: 6px 14px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--text); cursor: pointer; }
button.copy:hover { border-color: var(--accent); color: var(--accent); }
.status { font-size: 0.88rem; color: var(--good); min-height: 1.2em; }
.vh { position: absolute !important; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; }
@media (max-width: 600px) {
  nav.toc ul { flex-wrap: nowrap; overflow-x: auto; -webkit-overflow-scrolling: touch; white-space: nowrap; gap: 16px; }
  h1 { font-size: 1.6rem; }
  header.site { padding-top: 28px; }
  th, td { padding: 7px 9px; }
  table { font-size: 0.86rem; }
}
</style>
</head>
<body>
<header class="site">
  <div class="wrap">
    <h1>Pev Leaderboard</h1>
    <p class="lede">Accuracy, calibration and paired comparisons on Pev-Bench: seven families of short decision questions a personal agent with long-term memory has to answer (which option the user would pick, whether an action needs approval, whether a memory applies, whether an action relies on a forgotten fact, whether a fact may be shared, how urgently to notify, and where to route a request).</p>
    <ul class="links" aria-label="Project links">
      <li><a href="https://github.com/EnvLoop/Pev/blob/main/docs/tech-report/pev.pdf">Paper</a></li>
      <li><a href="https://huggingface.co/EnvLoop/Pev-27B-LoRA">Model</a></li>
      <li><a href="https://huggingface.co/datasets/EnvLoop/Pev-Bench">Dataset</a></li>
      <li><a href="https://github.com/EnvLoop/Pev">Code</a></li>
      <li><a href="https://huggingface.co/collections/EnvLoop/pev-6abf32b6b1940477ad4c52c3">Collection</a></li>
    </ul>
    <p class="note">EnvLoop Research. The model (Pev-27B, a LoRA adapter on Qwen3.8-27B) and the dataset (Pev-Bench) are gated and released for non-commercial research only.</p>
  </div>
</header>

<nav class="toc" aria-label="Sections">
  <ul>
    <li><a href="#test">TEST leaderboard</a></li>
    <li><a href="#paired">Paired comparisons</a></li>
    <li><a href="#families">Per family</a></li>
    <li><a href="#hidden">HIDDEN gate</a></li>
    <li><a href="#dev">DEV references</a></li>
    <li><a href="#caveats">Caveats</a></li>
    <li><a href="#figures">Figures</a></li>
    <li><a href="#cite">Citation</a></li>
  </ul>
</nav>

<main class="wrap">

<section id="test" aria-labelledby="test-h">
  <h2 id="test-h">TEST leaderboard</h2>
  <p>The public TEST set has <strong id="t-states"></strong> states from 720 new users and <strong id="t-scorable"></strong> scorable questions. Half A (<span id="t-a"></span>) was rendered to text by gpt-6-astra and half B (<span id="t-b"></span>) by Claude Opus 5.5 under the same rendering contract. TEST was pseudonymized before any model was scored, so the released TEST is exactly the scored TEST. Every model was run once and all predictions were frozen before scoring.</p>
  <p>The primary metric is family-macro accuracy: the mean of the seven per-family accuracies, with questions whose soft label has no unique answer excluded. The 6-family columns are the pre-registered sensitivity analysis without <code>pick_option</code>. Brier and ECE are family-macro averages; automation coverage is the share of questions answered alone at the threshold fitted on VAL for a 5% error budget.</p>
  <div class="table-scroll"><table id="tbl-leader"></table></div>
  <p class="note">* gpt-6-astra and Jev return probabilities through their APIs; these are used as returned (no temperature fitted on VAL), so their Brier and ECE are uncalibrated and they have no automation threshold (n/a). Accuracy does not depend on calibration. The local models are scored from label-token log-probabilities with a temperature fitted on VAL.</p>
</section>

<section id="paired" aria-labelledby="paired-h">
  <h2 id="paired-h">Paired comparisons</h2>
  <p>Pev-27B minus each other model, family-macro accuracy in points with a 95% confidence interval from a paired bootstrap over state clusters (10,000 resamples; <span id="p-clusters"></span> clusters on all of TEST). p-values are one-sided and Holm-adjusted over the five comparisons, separately for each column; 5 &times; 10<sup>&minus;4</sup> is the smallest value attainable with 10,000 resamples and five comparisons.</p>
  <div class="controls">
    <fieldset class="seg" id="seg-paired">
      <legend>Families</legend>
      <input type="radio" name="pfam" id="pfam7" value="p7" checked><label for="pfam7">7 families (primary)</label>
      <input type="radio" name="pfam" id="pfam6" value="p6"><label for="pfam6">6 families (without pick_option)</label>
    </fieldset>
  </div>
  <div class="table-scroll"><table id="tbl-paired"></table></div>
  <p class="note">The sensitivity analysis without <code>pick_option</code> agrees with the primary result in every comparison and half (same sign, all significant after Holm correction).</p>
</section>

<section id="families" aria-labelledby="fam-h">
  <h2 id="fam-h">Per family</h2>
  <div class="controls">
    <div class="field">
      <label for="famSel">Question family</label>
      <select id="famSel"></select>
    </div>
  </div>
  <p class="famdesc" id="famDesc"></p>
  <p class="note" id="famMeta"></p>
  <div class="table-scroll"><table id="tbl-fam"></table></div>
  <div class="callout" id="famPick" hidden><strong>pick_option is not solved.</strong> All six models score 0.413&ndash;0.480. A &ldquo;priciest option&rdquo; shortcut scores 0.347 on TEST against a chance level of 0.276 (+10.9 points over chance on half A, +2.9 on half B), and Pev-27B is not significantly ahead of gpt-6-astra or the base model on this family. No <code>pick_option</code> progress is claimed.</div>
</section>

<section id="hidden" aria-labelledby="hidden-h">
  <h2 id="hidden-h">HIDDEN one-shot gate</h2>
  <p>Before TEST existed, the frozen Pev-27B adapter was evaluated exactly once on a sealed, pre-registered HIDDEN set (<span id="h-states"></span> states, <span id="h-scorable"></span> scorable of <span id="h-questions"></span> questions; 446 of the states use rendering styles never seen in training). The comparison is with the zero-shot base model. The run produced aggregate-only outputs and the sealed set was deleted afterwards. <strong>HIDDEN is not released.</strong> <span id="h-gate"></span></p>
  <div class="stats" id="h-stats"></div>
  <div class="table-scroll"><table id="tbl-hidden"></table></div>
  <div class="table-scroll"><table id="tbl-hidden-safety"></table></div>
</section>

<section id="dev" aria-labelledby="dev-h">
  <h2 id="dev-h">DEV references</h2>
  <p>DEV (630 states, <span id="d-scorable"></span> scorable questions) was the model-selection set for Pev-27B, so its DEV score carries selection bias; HIDDEN and TEST confirm it. These numbers were computed on the DEV text before pseudonymization; the released DEV differs only in person names, contact details and user ids and was not re-scored, so re-running on it may give slightly different numbers.</p>
  <div class="table-scroll"><table id="tbl-dev"></table></div>
  <p class="note" id="d-paired"></p>
</section>

<section id="caveats" aria-labelledby="cav-h">
  <h2 id="cav-h">Caveats</h2>
  <ul class="caveats">
    <li><strong><code>pick_option</code> is essentially unsolved.</strong> It is the one family labelled by real behaviour (an item the user later rated 4 or higher). All six models score 0.41&ndash;0.48 on TEST; a &ldquo;priciest option&rdquo; shortcut scores 0.347 against a chance level of 0.276, so no <code>pick_option</code> progress is claimed. Pev-27B&rsquo;s VAL threshold also over-automates this family (realized error 37% on accepted TEST questions against the 5% budget): a deployment should always ask the user here.</li>
    <li><strong>Safety false negatives.</strong> Pev-27B has no <code>forgotten_violation</code> false negatives on TEST, but gpt-6-astra is slightly lower on the other two safety families (0.7% / 0% / 0% against 0% / 1.3% / 2.0% for <code>forgotten_violation</code> / <code>needs_approval</code> / <code>share_ok</code>). Rates per model are below (all TEST users; about 150 questions at risk per family).</li>
  </ul>
  <div class="table-scroll"><table id="tbl-safety"></table></div>
  <ul class="caveats">
    <li><strong>The text is LLM-rendered.</strong> Every state is rendered from structured facts under verbatim anchor checks, and memory facts are LLM-extracted. User histories are real public reviews, but approval rules, privacy levels, contacts and calendars are synthetic. Real user text and real rules will be messier.</li>
    <li><strong>Six of the seven families are rule-derived.</strong> High scores there show that the generator&rsquo;s rules can be learned from rendered text, including buried and overridden evidence and an unseen renderer (TEST half B), not general judgement about approvals or privacy.</li>
    <li><strong>Renderer familiarity.</strong> gpt-6-astra rendered the training, DEV and TEST-A text. On TEST no home advantage is visible: gpt-6-astra scores 0.876 on half A and 0.871 on half B, Pev-27B 0.913 and 0.917.</li>
    <li><strong>Not a safety system.</strong> Approval and sharing predictions must be backed by hard rules in an agent; the model is a fast, calibrated advisor, not an enforcement layer.</li>
  </ul>
</section>

<section id="figures" aria-labelledby="fig-h">
  <h2 id="fig-h">Figures</h2>
  <p class="note">From the technical report (CC-BY-4.0).</p>
  <div class="figs">
    <figure>
      <div class="img"><img src="figures/fig6_test_macro.png" width="1251" height="655" loading="lazy" alt="Grouped bar chart of TEST family-macro accuracy for six models on all of TEST, half A and half B. Pev-27B is highest in each group at about 0.91 to 0.92, followed by gpt-6-astra at about 0.87, Kev-27B about 0.82, Jev about 0.78 to 0.80, the Qwen3.8-27B base about 0.76 and Qwen3.5-4B about 0.63 to 0.68."></div>
      <figcaption>TEST family-macro accuracy per model, overall and per renderer half (A: gpt-6-astra, B: Claude Opus 5.5).</figcaption>
    </figure>
    <figure>
      <div class="img"><img src="figures/fig7_test_deltas.png" width="1343" height="633" loading="lazy" alt="Dot plot with confidence intervals of Pev-27B minus each model on TEST, in points, for all of TEST, half A and half B. All intervals lie above zero: about 4 points versus gpt-6-astra, 9 versus Kev-27B, 12 to 13 versus Jev, 15 versus the Qwen3.8-27B base and 24 to 29 versus Qwen3.5-4B."></div>
      <figcaption>Pev-27B minus each model on TEST, family-macro accuracy with paired 95% confidence intervals, overall and per half.</figcaption>
    </figure>
    <figure>
      <div class="img"><img src="figures/fig1_hidden_accuracy.png" width="1290" height="695" loading="lazy" alt="Dumbbell chart of per-family accuracy on HIDDEN for the Qwen3.8-27B base and Pev-27B with chance marks. Gains in points: apply memory +15.8, forgotten violation +4.4, needs approval +15.3, share ok +16.7, route +8.1, notify level +42.2, pick option +3.3, macro +15.1."></div>
      <figcaption>Per-family accuracy on HIDDEN for the base model (orange) and Pev-27B (blue), with chance marks; labels give the change in points.</figcaption>
    </figure>
    <figure>
      <div class="img"><img src="figures/fig4_hidden_automation.png" width="1065" height="644" loading="lazy" alt="Scatter plot of automation coverage against realized error per family on HIDDEN at the VAL-fitted thresholds, with a horizontal line at the 5% error budget. Pev-27B covers all questions in six families with errors below the budget, but its pick option point sits at about 0.51 coverage and 0.45 realized error."></div>
      <figcaption>Coverage and realized error per family on HIDDEN at the VAL-fitted thresholds; the line is the 5% error budget.</figcaption>
    </figure>
  </div>
</section>

</main>

<footer class="site" id="cite">
  <div class="wrap">
    <h2>Citation</h2>
    <div class="bibbar">
      <button type="button" class="copy" id="copyBib" aria-describedby="copyStatus">Copy BibTeX</button>
      <span class="status" id="copyStatus" role="status" aria-live="polite"></span>
    </div>
<pre class="bib" id="bib" tabindex="0">@techreport{envloop_pev,
  title       = {Pev: A Calibrated Fast-Decision Model for Personal Agents},
  author      = {{EnvLoop Research}},
  institution = {EnvLoop},
  url         = {https://github.com/EnvLoop/Pev}
}</pre>
    <p>Contact and removal requests: <a href="mailto:research@envloop.ai">research@envloop.ai</a></p>
    <p class="note">The content of this page and its figures are licensed under CC-BY-4.0. The model and the dataset are gated and have their own non-commercial research licences; see their cards.</p>
  </div>
</footer>

<script>
(function () {
  "use strict";
  var DATA = {"models":[{"key":"pev","name":"Pev-27B","local":true,"ours":true},{"key":"astra","name":"gpt-6-astra","local":false,"ours":false},{"key":"kev","name":"Kev-27B","local":true,"ours":false},{"key":"jev","name":"Jev (jev-1.13.0)","local":false,"ours":false},{"key":"base","name":"Qwen3.8-27B (base)","local":true,"ours":false},{"key":"qwen4b","name":"Qwen3.5-4B","local":true,"ours":false}],"families":["apply_memory","forgotten_violation","needs_approval","share_ok","route","notify_level","pick_option"],"safetyFamilies":["forgotten_violation","needs_approval","share_ok"],"chance":{"apply_memory":0.5,"forgotten_violation":0.5,"needs_approval":0.5,"share_ok":0.5,"route":0.167,"notify_level":0.25,"pick_option":0.276},"test":{"n":{"all":{"questions":2506,"scorable":2156,"states":980},"A":{"questions":1252,"scorable":1073,"states":490},"B":{"questions":1254,"scorable":1083,"states":490}},"clusters":970,"nScored":{"all":{"apply_memory":302,"forgotten_violation":301,"needs_approval":298,"share_ok":303,"route":339,"notify_level":313,"pick_option":300},"A":{"apply_memory":145,"forgotten_violation":152,"needs_approval":147,"share_ok":150,"route":169,"notify_level":155,"pick_option":155},"B":{"apply_memory":157,"forgotten_violation":149,"needs_approval":151,"share_ok":153,"route":170,"notify_level":158,"pick_option":145}},"rows":{"pev":{"m7":{"all":0.915,"A":0.913,"B":0.917},"m6":{"all":0.987,"A":0.987,"B":0.987},"brier":{"all":0.125,"A":0.123,"B":0.128},"ece":{"all":0.065,"A":0.069,"B":0.065},"cov":{"all":0.94,"A":0.93,"B":0.94},"fam":{"apply_memory":{"all":1.0,"A":1.0,"B":1.0},"forgotten_violation":{"all":0.997,"A":1.0,"B":0.993},"needs_approval":{"all":0.993,"A":0.993,"B":0.993},"share_ok":{"all":0.99,"A":0.993,"B":0.987},"route":{"all":0.988,"A":0.982,"B":0.994},"notify_level":{"all":0.955,"A":0.955,"B":0.956},"pick_option":{"all":0.48,"A":0.465,"B":0.497}},"fn":{"all":{"forgotten_violation":0.0,"needs_approval":1.3,"share_ok":2.0},"A":{"forgotten_violation":0.0,"needs_approval":1.4,"share_ok":1.3},"B":{"forgotten_violation":0.0,"needs_approval":1.3,"share_ok":2.6}}},"astra":{"m7":{"all":0.873,"A":0.876,"B":0.871},"m6":{"all":0.945,"A":0.946,"B":0.944},"brier":{"all":0.18,"A":0.177,"B":0.184},"ece":{"all":0.074,"A":0.081,"B":0.079},"cov":{"all":null,"A":null,"B":null},"fam":{"apply_memory":{"all":0.977,"A":0.979,"B":0.975},"forgotten_violation":{"all":0.993,"A":1.0,"B":0.987},"needs_approval":{"all":0.993,"A":0.993,"B":0.993},"share_ok":{"all":1.0,"A":1.0,"B":1.0},"route":{"all":0.909,"A":0.905,"B":0.912},"notify_level":{"all":0.799,"A":0.8,"B":0.797},"pick_option":{"all":0.443,"A":0.452,"B":0.434}},"fn":{"all":{"forgotten_violation":0.7,"needs_approval":0.0,"share_ok":0.0},"A":{"forgotten_violation":0.0,"needs_approval":0.0,"share_ok":0.0},"B":{"forgotten_violation":1.3,"needs_approval":0.0,"share_ok":0.0}}},"kev":{"m7":{"all":0.823,"A":0.825,"B":0.821},"m6":{"all":0.888,"A":0.885,"B":0.891},"brier":{"all":0.218,"A":0.22,"B":0.217},"ece":{"all":0.074,"A":0.086,"B":0.09},"cov":{"all":0.71,"A":0.7,"B":0.71},"fam":{"apply_memory":{"all":0.977,"A":0.986,"B":0.968},"forgotten_violation":{"all":0.997,"A":1.0,"B":0.993},"needs_approval":{"all":0.893,"A":0.905,"B":0.881},"share_ok":{"all":0.977,"A":0.967,"B":0.987},"route":{"all":0.909,"A":0.882,"B":0.935},"notify_level":{"all":0.575,"A":0.568,"B":0.582},"pick_option":{"all":0.433,"A":0.465,"B":0.4}},"fn":{"all":{"forgotten_violation":0.0,"needs_approval":2.0,"share_ok":4.6},"A":{"forgotten_violation":0.0,"needs_approval":1.4,"share_ok":6.7},"B":{"forgotten_violation":0.0,"needs_approval":2.6,"share_ok":2.6}}},"jev":{"m7":{"all":0.788,"A":0.779,"B":0.797},"m6":{"all":0.851,"A":0.837,"B":0.864},"brier":{"all":0.266,"A":0.276,"B":0.257},"ece":{"all":0.123,"A":0.128,"B":0.129},"cov":{"all":null,"A":null,"B":null},"fam":{"apply_memory":{"all":0.957,"A":0.945,"B":0.968},"forgotten_violation":{"all":0.997,"A":1.0,"B":0.993},"needs_approval":{"all":0.883,"A":0.878,"B":0.887},"share_ok":{"all":0.875,"A":0.833,"B":0.915},"route":{"all":0.876,"A":0.864,"B":0.888},"notify_level":{"all":0.518,"A":0.503,"B":0.532},"pick_option":{"all":0.413,"A":0.432,"B":0.393}},"fn":{"all":{"forgotten_violation":0.0,"needs_approval":4.0,"share_ok":21.9},"A":{"forgotten_violation":0.0,"needs_approval":6.8,"share_ok":28.0},"B":{"forgotten_violation":0.0,"needs_approval":1.3,"share_ok":15.8}}},"base":{"m7":{"all":0.762,"A":0.759,"B":0.766},"m6":{"all":0.818,"A":0.811,"B":0.824},"brier":{"all":0.264,"A":0.268,"B":0.26},"ece":{"all":0.067,"A":0.077,"B":0.076},"cov":{"all":0.56,"A":0.55,"B":0.58},"fam":{"apply_memory":{"all":0.844,"A":0.807,"B":0.879},"forgotten_violation":{"all":0.967,"A":0.967,"B":0.966},"needs_approval":{"all":0.819,"A":0.81,"B":0.828},"share_ok":{"all":0.868,"A":0.86,"B":0.876},"route":{"all":0.917,"A":0.905,"B":0.929},"notify_level":{"all":0.492,"A":0.516,"B":0.468},"pick_option":{"all":0.43,"A":0.445,"B":0.414}},"fn":{"all":{"forgotten_violation":6.0,"needs_approval":4.7,"share_ok":4.6},"A":{"forgotten_violation":6.6,"needs_approval":5.5,"share_ok":6.7},"B":{"forgotten_violation":5.3,"needs_approval":3.9,"share_ok":2.6}}},"qwen4b":{"m7":{"all":0.652,"A":0.627,"B":0.677},"m6":{"all":0.686,"A":0.66,"B":0.711},"brier":{"all":0.389,"A":0.405,"B":0.374},"ece":{"all":0.074,"A":0.088,"B":0.09},"cov":{"all":0.11,"A":0.11,"B":0.1},"fam":{"apply_memory":{"all":0.758,"A":0.683,"B":0.828},"forgotten_violation":{"all":0.711,"A":0.671,"B":0.752},"needs_approval":{"all":0.628,"A":0.605,"B":0.649},"share_ok":{"all":0.653,"A":0.647,"B":0.66},"route":{"all":0.791,"A":0.811,"B":0.771},"notify_level":{"all":0.575,"A":0.542,"B":0.608},"pick_option":{"all":0.45,"A":0.432,"B":0.469}},"fn":{"all":{"forgotten_violation":56.3,"needs_approval":5.4,"share_ok":47.7},"A":{"forgotten_violation":65.8,"needs_approval":5.5,"share_ok":49.3},"B":{"forgotten_violation":46.7,"needs_approval":5.3,"share_ok":46.1}}}},"paired":{"astra":{"p7":{"all":{"d":4.1,"lo":2.8,"hi":5.5,"p":"5 × 10⁻⁴"},"A":{"d":3.7,"lo":1.9,"hi":5.5,"p":"5 × 10⁻⁴"},"B":{"d":4.6,"lo":2.6,"hi":6.6,"p":"5 × 10⁻⁴"}},"p6":{"all":{"d":4.2,"lo":3.1,"hi":5.3,"p":"5 × 10⁻⁴"},"A":{"d":4.1,"lo":2.6,"hi":5.6,"p":"5 × 10⁻⁴"},"B":{"d":4.3,"lo":2.8,"hi":5.9,"p":"5 × 10⁻⁴"}},"fam":{"apply_memory":{"all":{"d":2.3,"lo":0.7,"hi":4.2},"A":{"d":2.1,"lo":0.0,"hi":4.7},"B":{"d":2.5,"lo":0.6,"hi":5.3}},"forgotten_violation":{"all":{"d":0.3,"lo":0.0,"hi":1.1},"A":{"d":0.0,"lo":0.0,"hi":0.0},"B":{"d":0.7,"lo":0.0,"hi":2.2}},"needs_approval":{"all":{"d":0.0,"lo":-1.3,"hi":1.3},"A":{"d":0.0,"lo":-2.0,"hi":2.0},"B":{"d":0.0,"lo":-1.9,"hi":1.9}},"share_ok":{"all":{"d":-1.0,"lo":-2.3,"hi":0.0},"A":{"d":-0.7,"lo":-2.2,"hi":0.0},"B":{"d":-1.3,"lo":-3.4,"hi":0.0}},"route":{"all":{"d":8.0,"lo":5.0,"hi":11.2},"A":{"d":7.7,"lo":3.2,"hi":12.5},"B":{"d":8.2,"lo":4.4,"hi":12.6}},"notify_level":{"all":{"d":15.7,"lo":10.5,"hi":20.9},"A":{"d":15.5,"lo":8.2,"hi":23.1},"B":{"d":15.8,"lo":8.7,"hi":23.0}},"pick_option":{"all":{"d":3.7,"lo":-3.1,"hi":10.3},"A":{"d":1.3,"lo":-7.4,"hi":10.0},"B":{"d":6.2,"lo":-4.1,"hi":16.8}}}},"kev":{"p7":{"all":{"d":9.2,"lo":7.8,"hi":10.5,"p":"5 × 10⁻⁴"},"A":{"d":8.8,"lo":6.9,"hi":10.7,"p":"5 × 10⁻⁴"},"B":{"d":9.6,"lo":7.7,"hi":11.5,"p":"5 × 10⁻⁴"}},"p6":{"all":{"d":9.9,"lo":8.7,"hi":11.2,"p":"5 × 10⁻⁴"},"A":{"d":10.3,"lo":8.5,"hi":12.1,"p":"5 × 10⁻⁴"},"B":{"d":9.6,"lo":7.8,"hi":11.4,"p":"5 × 10⁻⁴"}},"fam":{"apply_memory":{"all":{"d":2.3,"lo":0.7,"hi":4.2},"A":{"d":1.4,"lo":0.0,"hi":3.6},"B":{"d":3.2,"lo":0.6,"hi":6.1}},"forgotten_violation":{"all":{"d":0.0,"lo":0.0,"hi":0.0},"A":{"d":0.0,"lo":0.0,"hi":0.0},"B":{"d":0.0,"lo":0.0,"hi":0.0}},"needs_approval":{"all":{"d":10.1,"lo":6.7,"hi":13.8},"A":{"d":8.8,"lo":4.6,"hi":13.7},"B":{"d":11.3,"lo":6.1,"hi":16.8}},"share_ok":{"all":{"d":1.3,"lo":0.0,"hi":3.0},"A":{"d":2.7,"lo":0.6,"hi":5.5},"B":{"d":0.0,"lo":-1.9,"hi":1.9}},"route":{"all":{"d":8.0,"lo":5.0,"hi":11.1},"A":{"d":10.1,"lo":5.5,"hi":15.1},"B":{"d":5.9,"lo":2.2,"hi":10.1}},"notify_level":{"all":{"d":38.0,"lo":32.4,"hi":43.8},"A":{"d":38.7,"lo":30.7,"hi":47.0},"B":{"d":37.3,"lo":29.4,"hi":45.3}},"pick_option":{"all":{"d":4.7,"lo":-1.4,"hi":10.5},"A":{"d":0.0,"lo":-8.3,"hi":8.1},"B":{"d":9.7,"lo":0.7,"hi":18.7}}}},"jev":{"p7":{"all":{"d":12.7,"lo":11.2,"hi":14.1,"p":"5 × 10⁻⁴"},"A":{"d":13.3,"lo":11.3,"hi":15.4,"p":"5 × 10⁻⁴"},"B":{"d":12.0,"lo":10.0,"hi":14.2,"p":"5 × 10⁻⁴"}},"p6":{"all":{"d":13.7,"lo":12.2,"hi":15.1,"p":"5 × 10⁻⁴"},"A":{"d":15.0,"lo":12.9,"hi":17.2,"p":"5 × 10⁻⁴"},"B":{"d":12.3,"lo":10.4,"hi":14.2,"p":"5 × 10⁻⁴"}},"fam":{"apply_memory":{"all":{"d":4.3,"lo":2.2,"hi":6.7},"A":{"d":5.5,"lo":2.1,"hi":9.6},"B":{"d":3.2,"lo":0.7,"hi":6.1}},"forgotten_violation":{"all":{"d":0.0,"lo":0.0,"hi":0.0},"A":{"d":0.0,"lo":0.0,"hi":0.0},"B":{"d":0.0,"lo":0.0,"hi":0.0}},"needs_approval":{"all":{"d":11.1,"lo":7.5,"hi":14.9},"A":{"d":11.6,"lo":6.6,"hi":16.9},"B":{"d":10.6,"lo":5.6,"hi":15.9}},"share_ok":{"all":{"d":11.6,"lo":8.0,"hi":15.3},"A":{"d":16.0,"lo":10.4,"hi":22.1},"B":{"d":7.2,"lo":3.4,"hi":11.6}},"route":{"all":{"d":11.2,"lo":7.8,"hi":14.8},"A":{"d":11.8,"lo":6.8,"hi":17.1},"B":{"d":10.6,"lo":5.9,"hi":15.7}},"notify_level":{"all":{"d":43.8,"lo":38.1,"hi":49.4},"A":{"d":45.2,"lo":36.9,"hi":53.6},"B":{"d":42.4,"lo":34.5,"hi":50.4}},"pick_option":{"all":{"d":6.7,"lo":1.0,"hi":12.5},"A":{"d":3.2,"lo":-4.1,"hi":10.3},"B":{"d":10.3,"lo":1.4,"hi":19.4}}}},"base":{"p7":{"all":{"d":15.2,"lo":13.6,"hi":16.8,"p":"5 × 10⁻⁴"},"A":{"d":15.4,"lo":13.0,"hi":17.9,"p":"5 × 10⁻⁴"},"B":{"d":15.1,"lo":12.9,"hi":17.3,"p":"5 × 10⁻⁴"}},"p6":{"all":{"d":16.9,"lo":15.4,"hi":18.6,"p":"5 × 10⁻⁴"},"A":{"d":17.6,"lo":15.2,"hi":20.0,"p":"5 × 10⁻⁴"},"B":{"d":16.3,"lo":14.1,"hi":18.6,"p":"5 × 10⁻⁴"}},"fam":{"apply_memory":{"all":{"d":15.6,"lo":11.6,"hi":19.9},"A":{"d":19.3,"lo":13.1,"hi":26.0},"B":{"d":12.1,"lo":7.3,"hi":17.4}},"forgotten_violation":{"all":{"d":3.0,"lo":1.3,"hi":5.1},"A":{"d":3.3,"lo":0.7,"hi":6.4},"B":{"d":2.7,"lo":0.6,"hi":5.6}},"needs_approval":{"all":{"d":17.4,"lo":13.1,"hi":22.0},"A":{"d":18.4,"lo":12.2,"hi":25.0},"B":{"d":16.6,"lo":10.6,"hi":22.9}},"share_ok":{"all":{"d":12.2,"lo":8.4,"hi":16.2},"A":{"d":13.3,"lo":8.2,"hi":19.2},"B":{"d":11.1,"lo":5.6,"hi":16.9}},"route":{"all":{"d":7.1,"lo":4.3,"hi":10.0},"A":{"d":7.7,"lo":3.5,"hi":12.2},"B":{"d":6.5,"lo":2.6,"hi":10.7}},"notify_level":{"all":{"d":46.3,"lo":40.5,"hi":52.2},"A":{"d":43.9,"lo":35.6,"hi":52.1},"B":{"d":48.7,"lo":40.3,"hi":56.9}},"pick_option":{"all":{"d":5.0,"lo":-1.2,"hi":11.2},"A":{"d":1.9,"lo":-6.9,"hi":10.5},"B":{"d":8.3,"lo":-0.6,"hi":17.2}}}},"qwen4b":{"p7":{"all":{"d":26.3,"lo":24.1,"hi":28.4,"p":"5 × 10⁻⁴"},"A":{"d":28.5,"lo":25.5,"hi":31.6,"p":"5 × 10⁻⁴"},"B":{"d":24.1,"lo":21.1,"hi":27.0,"p":"5 × 10⁻⁴"}},"p6":{"all":{"d":30.1,"lo":27.9,"hi":32.3,"p":"5 × 10⁻⁴"},"A":{"d":32.8,"lo":29.7,"hi":35.8,"p":"5 × 10⁻⁴"},"B":{"d":27.6,"lo":24.6,"hi":30.7,"p":"5 × 10⁻⁴"}},"fam":{"apply_memory":{"all":{"d":24.2,"lo":19.4,"hi":29.2},"A":{"d":31.7,"lo":24.2,"hi":39.4},"B":{"d":17.2,"lo":11.6,"hi":23.3}},"forgotten_violation":{"all":{"d":28.6,"lo":23.5,"hi":33.9},"A":{"d":32.9,"lo":25.5,"hi":40.6},"B":{"d":24.2,"lo":17.2,"hi":31.4}},"needs_approval":{"all":{"d":36.6,"lo":31.1,"hi":42.2},"A":{"d":38.8,"lo":30.6,"hi":47.0},"B":{"d":34.4,"lo":26.7,"hi":42.3}},"share_ok":{"all":{"d":33.7,"lo":28.4,"hi":39.1},"A":{"d":34.7,"lo":27.1,"hi":42.5},"B":{"d":32.7,"lo":25.3,"hi":40.3}},"route":{"all":{"d":19.8,"lo":15.5,"hi":24.1},"A":{"d":17.2,"lo":11.7,"hi":22.9},"B":{"d":22.4,"lo":16.0,"hi":29.0}},"notify_level":{"all":{"d":38.0,"lo":32.2,"hi":43.8},"A":{"d":41.3,"lo":33.1,"hi":49.4},"B":{"d":34.8,"lo":26.6,"hi":43.0}},"pick_option":{"all":{"d":3.0,"lo":-3.6,"hi":9.7},"A":{"d":3.2,"lo":-5.1,"hi":11.4},"B":{"d":2.8,"lo":-7.7,"hi":13.3}}}}},"holmP":["5 × 10⁻⁴"]},"hidden":{"states":960,"questions":2456,"scorable":2106,"clusters":934,"base":0.754,"cand":0.905,"d":15.1,"lo":13.5,"hi":16.8,"p":"1 × 10⁻⁴","mcnemar":{"candOnly":359,"baseOnly":48,"p":"5.7 × 10⁻⁶⁰"},"brier":[0.28,0.135],"ece":[0.068,0.067],"cov":[0.54,0.93],"accepted":[1161,1959],"fam":{"apply_memory":{"n":303,"base":0.842,"cand":1.0,"d":15.8,"lo":11.8,"hi":20.0,"covBase":0.68,"covCand":1.0,"errBase":0.044,"errCand":0.0},"forgotten_violation":{"n":293,"base":0.956,"cand":1.0,"d":4.4,"lo":2.2,"hi":6.9,"covBase":0.81,"covCand":1.0,"errBase":0.004,"errCand":0.0},"needs_approval":{"n":308,"base":0.834,"cand":0.987,"d":15.3,"lo":11.4,"hi":19.5,"covBase":0.71,"covCand":1.0,"errBase":0.106,"errCand":0.013},"share_ok":{"n":306,"base":0.807,"cand":0.974,"d":16.7,"lo":12.5,"hi":21.0,"covBase":0.62,"covCand":1.0,"errBase":0.026,"errCand":0.026},"route":{"n":322,"base":0.898,"cand":0.978,"d":8.1,"lo":5.1,"hi":11.2,"covBase":0.84,"covCand":1.0,"errBase":0.074,"errCand":0.022},"notify_level":{"n":275,"base":0.545,"cand":0.967,"d":42.2,"lo":35.7,"hi":48.3,"covBase":0.06,"covCand":1.0,"errBase":0.312,"errCand":0.033},"pick_option":{"n":299,"base":0.398,"cand":0.431,"d":3.3,"lo":-3.1,"hi":9.6,"covBase":0.09,"covCand":0.51,"errBase":0.269,"errCand":0.447}},"safety":{"forgotten_violation":{"base":8.8,"cand":0.0,"d":-8.8,"lo":-13.5,"hi":-4.7,"n":148},"needs_approval":{"base":1.3,"cand":0.0,"d":-1.3,"lo":-3.3,"hi":0.0,"n":153},"share_ok":{"base":11.0,"cand":3.9,"d":-7.1,"lo":-11.7,"hi":-2.6,"n":154}},"gatePassed":true,"checksPassed":7,"checksTotal":7},"dev":{"scorable":1409,"order":["pev","astra","kev","jev","base","qwen4b"],"rows":{"pev":{"m7":0.907,"brier":0.131,"ece":0.066,"fam":{"apply_memory":1.0,"forgotten_violation":1.0,"needs_approval":0.995,"share_ok":0.995,"route":0.99,"notify_level":0.912,"pick_option":0.457},"fn":{"forgotten_violation":0.0,"needs_approval":0.0,"share_ok":1.0}},"astra":{"m7":0.881,"brier":0.178,"ece":0.075,"fam":{"apply_memory":0.977,"forgotten_violation":0.995,"needs_approval":0.995,"share_ok":0.971,"route":0.954,"notify_level":0.83,"pick_option":0.442},"fn":{"forgotten_violation":1.0,"needs_approval":0.0,"share_ok":1.0}},"kev":{"m7":0.824,"brier":0.213,"ece":0.075,"fam":{"apply_memory":0.972,"forgotten_violation":1.0,"needs_approval":0.954,"share_ok":0.962,"route":0.909,"notify_level":0.577,"pick_option":0.396},"fn":{"forgotten_violation":0.0,"needs_approval":0.0,"share_ok":7.7}},"jev":{"m7":0.785,"brier":0.279,"ece":0.12,"fam":{"apply_memory":0.944,"forgotten_violation":0.995,"needs_approval":0.923,"share_ok":0.837,"route":0.868,"notify_level":0.552,"pick_option":0.376},"fn":{"forgotten_violation":0.0,"needs_approval":0.0,"share_ok":27.9}},"base":{"m7":0.766,"brier":0.261,"ece":0.074,"fam":{"apply_memory":0.85,"forgotten_violation":0.966,"needs_approval":0.892,"share_ok":0.847,"route":0.919,"notify_level":0.515,"pick_option":0.376},"fn":{"forgotten_violation":6.9,"needs_approval":2.1,"share_ok":4.8}},"qwen4b":{"m7":0.638,"brier":0.408,"ece":0.068,"fam":{"apply_memory":0.737,"forgotten_violation":0.737,"needs_approval":0.686,"share_ok":0.636,"route":0.731,"notify_level":0.598,"pick_option":0.34},"fn":{"forgotten_violation":52.0,"needs_approval":6.2,"share_ok":51.9}}},"paired":{"jev":{"d":12.2,"lo":10.4,"hi":14.0,"p":"< 10⁻⁴","clusters":625},"astra":{"d":2.7,"lo":1.1,"hi":4.2,"p":"7 × 10⁻⁴","clusters":625}}}};

  var FAM_INFO = {
    apply_memory: { label: "apply_memory", type: "yes/no", desc: "Is a remembered fact relevant to the current request? Guards against over-personalisation." },
    forgotten_violation: { label: "forgotten_violation", type: "yes/no", desc: "Would the candidate action rely on a fact the user asked the agent to forget?" },
    needs_approval: { label: "needs_approval", type: "yes/no", desc: "Does the action need the user's approval under the spend threshold in force, the new-recipient rule and the irreversible-action rule?" },
    share_ok: { label: "share_ok", type: "yes/no", desc: "May a fact be shared with this recipient, given the recipient's clearance and the fact's privacy level?" },
    route: { label: "route", type: "choice (6 labels)", desc: "Which connected service should handle the request, or should the agent ask the user?" },
    notify_level: { label: "notify_level", type: "score (4 levels)", desc: "How urgently to notify (silent, digest, notify, interrupt), with quiet hours, a meeting cap and VIP bypass." },
    pick_option: { label: "pick_option", type: "choice", desc: "Which candidate item would the user pick? Labelled by real later behaviour (an item the user rated 4 or higher)." }
  };

  function el(tag, attrs, children) {
    var e = document.createElement(tag);
    if (attrs) {
      for (var k in attrs) {
        if (!Object.prototype.hasOwnProperty.call(attrs, k)) continue;
        if (k === "text") e.textContent = attrs[k];
        else if (k === "html") e.innerHTML = attrs[k];
        else if (k === "cls") e.className = attrs[k];
        else e.setAttribute(k, attrs[k]);
      }
    }
    (children || []).forEach(function (c) { if (c) e.appendChild(typeof c === "string" ? document.createTextNode(c) : c); });
    return e;
  }
  function setText(id, t) { var n = document.getElementById(id); if (n) n.textContent = t; }
  function int(n) { return n.toLocaleString("en-US"); }
  var MINUS = "−";
  function f3(x) { return x == null ? "n/a" : x.toFixed(3); }
  function f2(x) { return x == null ? "n/a" : x.toFixed(2); }
  function pt(x) {
    if (x > 0) return "+" + x.toFixed(1);
    if (x < 0) return MINUS + Math.abs(x).toFixed(1);
    return "0.0";
  }
  function dci(o) { return pt(o.d) + " [" + pt(o.lo) + ", " + pt(o.hi) + "]"; }
  function pc(x) { return x.toFixed(1) + "%"; }

  var models = DATA.models;
  var byKey = {};
  models.forEach(function (m) { byKey[m.key] = m; });
  var T = DATA.test;

  function nameCell(m, extraTag) {
    var th = el("th", { scope: "row", cls: "l sticky" }, [m.name]);
    if (extraTag) th.appendChild(el("span", { cls: "tag", text: extraTag }));
    return th;
  }
  function headRow(cells) {
    var tr = el("tr");
    cells.forEach(function (c) {
      var a = { scope: c.scope || "col", cls: c.cls || "num" };
      if (c.colspan) a.colspan = c.colspan;
      if (c.rowspan) a.rowspan = c.rowspan;
      if (c.title) a.title = c.title;
      tr.appendChild(el("th", a, [c.t]));
    });
    return tr;
  }
  function td(text, cls) { return el("td", { cls: "num " + (cls || "") }, [text]); }

  // ---------- header numbers ----------
  setText("t-states", int(T.n.all.states));
  setText("t-scorable", int(T.n.all.scorable));
  setText("t-a", int(T.n.A.states) + " states, " + int(T.n.A.scorable) + " scorable");
  setText("t-b", int(T.n.B.states) + " states, " + int(T.n.B.scorable) + " scorable");
  setText("p-clusters", int(T.clusters));

  // ---------- leaderboard ----------
  (function () {
    var t = document.getElementById("tbl-leader");
    t.appendChild(el("caption", { text: "TEST, sorted by family-macro accuracy over all users (7 families). A: gpt-6-astra-rendered half; B: Claude Opus 5.5-rendered half." }));
    var thead = el("thead");
    thead.appendChild(headRow([
      { t: "#", rowspan: 2, cls: "num" },
      { t: "Model", rowspan: 2, cls: "l sticky" },
      { t: "7 families (primary)", colspan: 3, cls: "grp", scope: "colgroup" },
      { t: "6 families (without pick_option)", colspan: 3, cls: "grp", scope: "colgroup" },
      { t: "Calibration and automation (7 families, all)", colspan: 3, cls: "grp", scope: "colgroup" }
    ]));
    thead.appendChild(headRow([
      { t: "All", cls: "num bl" }, { t: "A half" }, { t: "B half" },
      { t: "All", cls: "num bl" }, { t: "A half" }, { t: "B half" },
      { t: "Brier ↓", cls: "num bl", title: "Family-macro Brier score; lower is better" },
      { t: "ECE ↓", title: "Family-macro expected calibration error; lower is better" },
      { t: "Coverage @5%", title: "Share of questions answered alone at the VAL threshold for a 5% error budget" }
    ]));
    t.appendChild(thead);
    var tb = el("tbody");
    models.forEach(function (m, i) {
      var r = T.rows[m.key];
      var tr = el("tr", { cls: m.ours ? "ours" : "" });
      tr.appendChild(td(String(i + 1), "dim"));
      tr.appendChild(nameCell(m, m.local ? null : "API"));
      tr.appendChild(td(f3(r.m7.all), "bl"));
      tr.appendChild(td(f3(r.m7.A)));
      tr.appendChild(td(f3(r.m7.B)));
      tr.appendChild(td(f3(r.m6.all), "bl"));
      tr.appendChild(td(f3(r.m6.A)));
      tr.appendChild(td(f3(r.m6.B)));
      var star = m.local ? "" : "*";
      tr.appendChild(td(f3(r.brier.all) + star, "bl"));
      tr.appendChild(td(f3(r.ece.all) + star));
      tr.appendChild(td(f2(r.cov.all)));
      tb.appendChild(tr);
    });
    t.appendChild(tb);
  })();

  // ---------- paired ----------
  function renderPaired(which) {
    var t = document.getElementById("tbl-paired");
    t.textContent = "";
    var label = which === "p7" ? "7 families (primary)" : "6 families, without pick_option (sensitivity)";
    t.appendChild(el("caption", { text: "Pev-27B minus model, family-macro accuracy in points [95% CI]; " + label + "." }));
    var thead = el("thead");
    thead.appendChild(headRow([
      { t: "Pev-27B minus", cls: "l sticky" },
      { t: "All users" }, { t: "A half (gpt-6-astra)" }, { t: "B half (Claude Opus 5.5)" },
      { t: "Holm-adjusted p (all / A / B)" }
    ]));
    t.appendChild(thead);
    var tb = el("tbody");
    models.forEach(function (m) {
      if (m.ours) return;
      var p = T.paired[m.key][which];
      var tr = el("tr");
      tr.appendChild(nameCell(m, null));
      tr.appendChild(td(dci(p.all)));
      tr.appendChild(td(dci(p.A)));
      tr.appendChild(td(dci(p.B)));
      var ps = [p.all.p, p.A.p, p.B.p];
      tr.appendChild(td(ps[0] === ps[1] && ps[1] === ps[2] ? ps[0] + " (each)" : ps.join(" / ")));
      tb.appendChild(tr);
    });
    t.appendChild(tb);
  }
  renderPaired("p7");
  Array.prototype.forEach.call(document.querySelectorAll('input[name="pfam"]'), function (r) {
    r.addEventListener("change", function () { if (r.checked) renderPaired(r.value); });
  });

  // ---------- per family ----------
  var sel = document.getElementById("famSel");
  DATA.families.forEach(function (f) {
    sel.appendChild(el("option", { value: f, text: f }));
  });
  function chanceText(f) {
    var c = DATA.chance[f];
    if (f === "route") return "1/6 (" + c.toFixed(3) + ")";
    if (f === "pick_option") return c.toFixed(3) + " on TEST";
    return c.toFixed(c === 0.5 ? 1 : 2);
  }
  function renderFam(f) {
    var info = FAM_INFO[f];
    var d = document.getElementById("famDesc");
    d.textContent = "";
    d.appendChild(el("strong", { text: info.label }));
    d.appendChild(document.createTextNode(" (" + info.type + "). " + info.desc));
    setText("famMeta", "Chance level: " + chanceText(f) + ". Scorable questions (same for every model): all " +
      int(T.nScored.all[f]) + ", A half " + int(T.nScored.A[f]) + ", B half " + int(T.nScored.B[f]) + ".");
    var t = document.getElementById("tbl-fam");
    t.textContent = "";
    t.appendChild(el("caption", { text: "TEST accuracy on " + f + ", sorted by accuracy over all users; last column: Pev-27B minus model on this family, points [95% CI], not adjusted for multiple comparisons." }));
    var thead = el("thead");
    thead.appendChild(headRow([
      { t: "Model", cls: "l sticky" }, { t: "All" }, { t: "A half" }, { t: "B half" },
      { t: "Pev-27B minus model (all)", cls: "num bl" }
    ]));
    t.appendChild(thead);
    var tb = el("tbody");
    var ms = models.slice().sort(function (a, b) { return T.rows[b.key].fam[f].all - T.rows[a.key].fam[f].all; });
    ms.forEach(function (m) {
      var r = T.rows[m.key].fam[f];
      var tr = el("tr", { cls: m.ours ? "ours" : "" });
      tr.appendChild(nameCell(m, m.local ? null : "API"));
      tr.appendChild(td(f3(r.all)));
      tr.appendChild(td(f3(r.A)));
      tr.appendChild(td(f3(r.B)));
      tr.appendChild(td(m.ours ? "—" : dci(T.paired[m.key].fam[f].all), "bl"));
      tb.appendChild(tr);
    });
    var c = DATA.chance[f];
    var cr = el("tr", { cls: "chance" });
    cr.appendChild(el("th", { scope: "row", cls: "l sticky" }, ["Chance"]));
    cr.appendChild(td(c.toFixed(3)));
    cr.appendChild(td(""));
    cr.appendChild(td(""));
    cr.appendChild(td("", "bl"));
    tb.appendChild(cr);
    t.appendChild(tb);
    document.getElementById("famPick").hidden = f !== "pick_option";
  }
  sel.addEventListener("change", function () { renderFam(sel.value); });
  renderFam(sel.value || DATA.families[0]);

  // ---------- HIDDEN ----------
  (function () {
    var H = DATA.hidden;
    setText("h-states", int(H.states));
    setText("h-scorable", int(H.scorable));
    setText("h-questions", int(H.questions));
    setText("h-gate", "All " + H.checksTotal + " pre-registered gate checks passed.");
    var stats = document.getElementById("h-stats");
    function stat(k, v, s) {
      stats.appendChild(el("div", { cls: "stat" }, [el("div", { cls: "k", text: k }), el("div", { cls: "v", text: v }), el("div", { cls: "s", text: s })]));
    }
    stat("Family-macro accuracy", f3(H.base) + " → " + f3(H.cand), "Qwen3.8-27B (base) → Pev-27B");
    stat("Difference", pt(H.d) + " points", "95% CI [" + pt(H.lo) + ", " + pt(H.hi) + "]; one-sided p = " + H.p + " (" + int(H.clusters) + " state clusters)");
    stat("McNemar", H.mcnemar.candOnly + " vs " + H.mcnemar.baseOnly, "questions correct only for Pev-27B vs only for the base; exact p = " + H.mcnemar.p);
    stat("Automation coverage at 5% error", f2(H.cov[0]) + " → " + f2(H.cov[1]), "accepted questions " + int(H.accepted[0]) + " → " + int(H.accepted[1]) + "; Brier " + f3(H.brier[0]) + " → " + f3(H.brier[1]));

    var t = document.getElementById("tbl-hidden");
    t.appendChild(el("caption", { text: "HIDDEN per family: Qwen3.8-27B (base) vs Pev-27B. Coverage and realized error at the VAL-fitted thresholds (5% error budget)." }));
    var thead = el("thead");
    thead.appendChild(headRow([
      { t: "Family", cls: "l sticky" }, { t: "n scored" }, { t: "Base" }, { t: "Pev-27B" },
      { t: "Difference, points [95% CI]" }, { t: "Coverage base / ours", cls: "num bl" }, { t: "Realized error base / ours" }
    ]));
    t.appendChild(thead);
    var tb = el("tbody");
    DATA.families.forEach(function (f) {
      var r = H.fam[f];
      var tr = el("tr");
      tr.appendChild(el("th", { scope: "row", cls: "l sticky" }, [f]));
      tr.appendChild(td(int(r.n)));
      tr.appendChild(td(f3(r.base)));
      tr.appendChild(td(f3(r.cand)));
      tr.appendChild(td(dci(r) + (r.lo <= 0 ? " n.s." : "")));
      tr.appendChild(td(f2(r.covBase) + " / " + f2(r.covCand), "bl"));
      tr.appendChild(td(f3(r.errBase) + " / " + f3(r.errCand)));
      tb.appendChild(tr);
    });
    var mr = el("tr", { cls: "ours" });
    mr.appendChild(el("th", { scope: "row", cls: "l sticky" }, ["Macro (7 families)"]));
    mr.appendChild(td(int(H.scorable)));
    mr.appendChild(td(f3(H.base)));
    mr.appendChild(td(f3(H.cand)));
    mr.appendChild(td(dci(H)));
    mr.appendChild(td(f2(H.cov[0]) + " / " + f2(H.cov[1]), "bl"));
    mr.appendChild(td(""));
    tb.appendChild(mr);
    t.appendChild(tb);

    var s = document.getElementById("tbl-hidden-safety");
    s.appendChild(el("caption", { text: "HIDDEN safety false-negative rates (questions at risk in brackets)." }));
    var sh = el("thead");
    sh.appendChild(headRow([
      { t: "Family", cls: "l sticky" }, { t: "Base" }, { t: "Pev-27B" }, { t: "Change, points [95% CI]" }
    ]));
    s.appendChild(sh);
    var sb = el("tbody");
    DATA.safetyFamilies.forEach(function (f) {
      var r = H.safety[f];
      var tr = el("tr");
      tr.appendChild(el("th", { scope: "row", cls: "l sticky" }, [f + " (" + int(r.n) + ")"]));
      tr.appendChild(td(pc(r.base)));
      tr.appendChild(td(pc(r.cand)));
      tr.appendChild(td(dci(r)));
      sb.appendChild(tr);
    });
    s.appendChild(sb);
  })();

  // ---------- DEV ----------
  (function () {
    var Dv = DATA.dev;
    setText("d-scorable", int(Dv.scorable));
    var t = document.getElementById("tbl-dev");
    t.appendChild(el("caption", { text: "DEV accuracy per family (pre-pseudonymization text), sorted by family-macro accuracy." }));
    var thead = el("thead");
    var hc = [{ t: "Model", cls: "l sticky" }, { t: "Macro" }];
    DATA.families.forEach(function (f, i) { hc.push({ t: f, cls: i === 0 ? "num bl" : "num" }); });
    hc.push({ t: "Brier ↓", cls: "num bl" }, { t: "ECE ↓" });
    thead.appendChild(headRow(hc));
    t.appendChild(thead);
    var tb = el("tbody");
    Dv.order.forEach(function (k) {
      var m = byKey[k], r = Dv.rows[k];
      var tr = el("tr", { cls: m.ours ? "ours" : "" });
      tr.appendChild(nameCell(m, m.local ? null : "API"));
      tr.appendChild(td(f3(r.m7)));
      DATA.families.forEach(function (f, i) { tr.appendChild(td(f3(r.fam[f]), i === 0 ? "bl" : "")); });
      var star = m.local ? "" : "*";
      tr.appendChild(td(f3(r.brier) + star, "bl"));
      tr.appendChild(td(f3(r.ece) + star));
      tb.appendChild(tr);
    });
    t.appendChild(tb);
    var a = Dv.paired.astra, j = Dv.paired.jev;
    setText("d-paired", "Paired over " + int(a.clusters) + " DEV state clusters: Pev-27B minus gpt-6-astra " + dci(a) +
      " points (one-sided p = " + a.p + "); minus Jev (jev-1.13.0) " + dci(j) + " points (one-sided p " + (j.p.charAt(0) === "<" ? j.p : "= " + j.p) +
      "). * Uncalibrated: API probabilities used as returned.");
  })();

  // ---------- safety table (TEST) ----------
  (function () {
    var t = document.getElementById("tbl-safety");
    t.appendChild(el("caption", { text: "TEST safety false-negative rates, all users (lower is better)." }));
    var thead = el("thead");
    var hc = [{ t: "Model", cls: "l sticky" }];
    DATA.safetyFamilies.forEach(function (f) { hc.push({ t: f }); });
    thead.appendChild(headRow(hc));
    t.appendChild(thead);
    var tb = el("tbody");
    models.forEach(function (m) {
      var r = T.rows[m.key].fn.all;
      var tr = el("tr", { cls: m.ours ? "ours" : "" });
      tr.appendChild(nameCell(m, m.local ? null : "API"));
      DATA.safetyFamilies.forEach(function (f) { tr.appendChild(td(pc(r[f]))); });
      tb.appendChild(tr);
    });
    t.appendChild(tb);
  })();

  // ---------- copy BibTeX ----------
  (function () {
    var btn = document.getElementById("copyBib");
    var status = document.getElementById("copyStatus");
    var bib = document.getElementById("bib");
    function done(ok) {
      status.textContent = ok ? "Copied." : "Copy failed; select the text and copy it manually.";
      setTimeout(function () { status.textContent = ""; }, 3000);
    }
    function fallback(text) {
      try {
        var ta = document.createElement("textarea");
        ta.value = text;
        ta.setAttribute("readonly", "");
        ta.style.position = "fixed";
        ta.style.opacity = "0";
        document.body.appendChild(ta);
        ta.select();
        var ok = document.execCommand("copy");
        document.body.removeChild(ta);
        done(ok);
      } catch (e) { done(false); }
    }
    btn.addEventListener("click", function () {
      var text = bib.textContent;
      try {
        if (navigator.clipboard && navigator.clipboard.writeText) {
          navigator.clipboard.writeText(text).then(function () { done(true); }, function () { fallback(text); });
        } else {
          fallback(text);
        }
      } catch (e) { fallback(text); }
    });
  })();
})();
</script>
</body>
</html>