Spaces:
Running
Running
Download index.html from EnvLoop/Pev-Leaderboard: direct link, hf CLI and curl.
- Browser
- Download file 52.1 kB
-
https://huggingface.co/spaces/EnvLoop/Pev-Leaderboard/resolve/main/index.html
- Command line
-
hf download hf://spaces/EnvLoop/Pev-Leaderboard/index.html
-
curl -L -o index.html https://huggingface.co/spaces/EnvLoop/Pev-Leaderboard/resolve/main/index.html
52.1 kB
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1"> | |
| <title>Pev Leaderboard</title> | |
| <meta name="description" content="Leaderboard for Pev-Bench: family-macro accuracy, paired comparisons, calibration and automation coverage for personal-agent decision questions."> | |
| <meta name="color-scheme" content="light dark"> | |
| <style> | |
| :root { | |
| --bg: #f7f7f5; | |
| --surface: #ffffff; | |
| --surface-2: #f0f0ed; | |
| --text: #1c1d1f; | |
| --muted: #5d6066; | |
| --border: #dcdcd7; | |
| --accent: #2a62c9; | |
| --accent-soft: #e8effb; | |
| --hl: #eef3fc; | |
| --hl-border: #2a62c9; | |
| --warn-bg: #fbf5e6; | |
| --warn-border: #d9b25c; | |
| --good: #1f7a4d; | |
| --code-bg: #f0f0ed; | |
| --figure-card: #fcfcfa; | |
| --focus: #2a62c9; | |
| --radius: 10px; | |
| --font: system-ui, -apple-system, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; | |
| --mono: ui-monospace, SFMono-Regular, Menlo, Consolas, "Liberation Mono", monospace; | |
| } | |
| @media (prefers-color-scheme: dark) { | |
| :root { | |
| --bg: #141517; | |
| --surface: #1c1d20; | |
| --surface-2: #25272b; | |
| --text: #e6e6e3; | |
| --muted: #a3a6ad; | |
| --border: #34363b; | |
| --accent: #7ea6f0; | |
| --accent-soft: #1f2a3d; | |
| --hl: #1d2638; | |
| --hl-border: #7ea6f0; | |
| --warn-bg: #2a2416; | |
| --warn-border: #8a6d2c; | |
| --good: #6cc196; | |
| --code-bg: #25272b; | |
| --figure-card: #fcfcfa; | |
| --focus: #9dbcf5; | |
| } | |
| } | |
| * { box-sizing: border-box; } | |
| html { -webkit-text-size-adjust: 100%; } | |
| body { | |
| margin: 0; | |
| background: var(--bg); | |
| color: var(--text); | |
| font-family: var(--font); | |
| font-size: 16px; | |
| line-height: 1.55; | |
| overflow-x: hidden; | |
| } | |
| a { color: var(--accent); } | |
| a:focus-visible, button:focus-visible, select:focus-visible, input:focus-visible + label { | |
| outline: 2px solid var(--focus); | |
| outline-offset: 2px; | |
| } | |
| .wrap { max-width: 1120px; margin: 0 auto; padding: 0 16px; } | |
| header.site { padding: 40px 0 24px; border-bottom: 1px solid var(--border); background: var(--surface); } | |
| h1 { font-size: 2rem; line-height: 1.2; margin: 0 0 8px; letter-spacing: -0.01em; } | |
| h2 { font-size: 1.35rem; margin: 0 0 8px; line-height: 1.3; } | |
| h3 { font-size: 1.05rem; margin: 24px 0 8px; } | |
| p { margin: 0 0 12px; } | |
| .lede { font-size: 1.05rem; color: var(--muted); max-width: 820px; } | |
| .links { display: flex; flex-wrap: wrap; gap: 8px; margin: 18px 0 12px; padding: 0; list-style: none; } | |
| .links a { | |
| display: inline-block; padding: 6px 14px; border: 1px solid var(--border); border-radius: 999px; | |
| text-decoration: none; color: var(--text); background: var(--surface); font-size: 0.95rem; | |
| } | |
| .links a:hover { border-color: var(--accent); color: var(--accent); } | |
| .note { font-size: 0.9rem; color: var(--muted); } | |
| nav.toc { position: sticky; top: 0; z-index: 5; background: var(--bg); border-bottom: 1px solid var(--border); } | |
| nav.toc ul { list-style: none; margin: 0 auto; padding: 8px 16px; display: flex; gap: 4px 16px; flex-wrap: wrap; max-width: 1120px; } | |
| nav.toc a { text-decoration: none; color: var(--muted); font-size: 0.9rem; } | |
| nav.toc a:hover { color: var(--accent); } | |
| main section { padding: 32px 0 8px; } | |
| main section, footer.site { scroll-margin-top: 56px; } | |
| .card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); padding: 16px; } | |
| .table-scroll { overflow-x: auto; -webkit-overflow-scrolling: touch; border: 1px solid var(--border); border-radius: var(--radius); background: var(--surface); margin: 12px 0; } | |
| table { border-collapse: separate; border-spacing: 0; width: 100%; font-size: 0.92rem; } | |
| caption { text-align: left; padding: 10px 12px; color: var(--muted); font-size: 0.88rem; caption-side: top; } | |
| th, td { padding: 8px 12px; border-bottom: 1px solid var(--border); text-align: right; white-space: nowrap; } | |
| th { font-weight: 600; background: var(--surface-2); } | |
| thead th { position: relative; } | |
| th.l, td.l { text-align: left; } | |
| tbody tr:last-child th, tbody tr:last-child td { border-bottom: 0; } | |
| td.num, th.num { font-variant-numeric: tabular-nums; } | |
| .sticky { position: sticky; left: 0; z-index: 1; background: var(--surface); } | |
| thead .sticky { background: var(--surface-2); z-index: 2; } | |
| tbody th { font-weight: 500; text-align: left; } | |
| tr.ours th, tr.ours td { background: var(--hl); } | |
| tr.ours th.sticky { box-shadow: inset 3px 0 0 var(--hl-border); } | |
| tr.ours th { font-weight: 650; } | |
| tr.chance th, tr.chance td { color: var(--muted); font-style: italic; } | |
| .grp { text-align: center; border-left: 1px solid var(--border); } | |
| .bl { border-left: 1px solid var(--border); } | |
| .tag { display: inline-block; font-size: 0.72rem; padding: 1px 6px; border-radius: 4px; border: 1px solid var(--border); color: var(--muted); font-weight: 500; margin-left: 6px; vertical-align: 1px; } | |
| .dim { color: var(--muted); } | |
| .controls { display: flex; flex-wrap: wrap; gap: 12px 24px; align-items: center; margin: 12px 0 4px; } | |
| fieldset.seg { border: 0; padding: 0; margin: 0; display: inline-flex; flex-wrap: wrap; gap: 0; } | |
| fieldset.seg legend { font-size: 0.9rem; color: var(--muted); margin-bottom: 4px; padding: 0; width: 100%; } | |
| .seg input { position: absolute; opacity: 0; width: 1px; height: 1px; } | |
| .seg label { padding: 6px 12px; border: 1px solid var(--border); background: var(--surface); cursor: pointer; font-size: 0.9rem; } | |
| .seg label:first-of-type { border-radius: 8px 0 0 8px; } | |
| .seg label:last-of-type { border-radius: 0 8px 8px 0; border-left: 0; } | |
| .seg input:checked + label { background: var(--accent-soft); border-color: var(--accent); color: var(--text); font-weight: 600; } | |
| .field { display: flex; flex-direction: column; gap: 4px; } | |
| .field label { font-size: 0.9rem; color: var(--muted); } | |
| select { font: inherit; font-size: 0.95rem; padding: 6px 10px; border-radius: 8px; border: 1px solid var(--border); background: var(--surface); color: var(--text); max-width: 100%; } | |
| .stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(210px, 1fr)); gap: 12px; margin: 16px 0; } | |
| .stat { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); padding: 14px 16px; } | |
| .stat .k { font-size: 0.85rem; color: var(--muted); } | |
| .stat .v { font-size: 1.35rem; font-weight: 650; font-variant-numeric: tabular-nums; margin-top: 2px; } | |
| .stat .s { font-size: 0.85rem; color: var(--muted); margin-top: 2px; } | |
| .famdesc { margin: 8px 0 0; font-size: 0.95rem; } | |
| .callout { background: var(--warn-bg); border: 1px solid var(--warn-border); border-radius: var(--radius); padding: 12px 14px; margin: 12px 0; font-size: 0.95rem; } | |
| ul.caveats { padding-left: 20px; margin: 8px 0; } | |
| ul.caveats li { margin-bottom: 10px; } | |
| code { font-family: var(--mono); font-size: 0.88em; background: var(--code-bg); padding: 1px 4px; border-radius: 4px; } | |
| .figs { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(100%, 460px), 1fr)); gap: 16px; margin-top: 12px; } | |
| figure { margin: 0; background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); overflow: hidden; } | |
| figure .img { background: var(--figure-card); padding: 8px; } | |
| figure img { display: block; width: 100%; height: auto; } | |
| figcaption { padding: 10px 14px 12px; font-size: 0.9rem; color: var(--muted); } | |
| footer.site { margin-top: 40px; padding: 32px 0 48px; border-top: 1px solid var(--border); background: var(--surface); } | |
| pre.bib { background: var(--code-bg); border: 1px solid var(--border); border-radius: var(--radius); padding: 12px 14px; overflow-x: auto; font-family: var(--mono); font-size: 0.85rem; line-height: 1.5; margin: 8px 0; } | |
| .bibbar { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; } | |
| button.copy { font: inherit; font-size: 0.9rem; padding: 6px 14px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--text); cursor: pointer; } | |
| button.copy:hover { border-color: var(--accent); color: var(--accent); } | |
| .status { font-size: 0.88rem; color: var(--good); min-height: 1.2em; } | |
| .vh { position: absolute ; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; } | |
| @media (max-width: 600px) { | |
| nav.toc ul { flex-wrap: nowrap; overflow-x: auto; -webkit-overflow-scrolling: touch; white-space: nowrap; gap: 16px; } | |
| h1 { font-size: 1.6rem; } | |
| header.site { padding-top: 28px; } | |
| th, td { padding: 7px 9px; } | |
| table { font-size: 0.86rem; } | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <header class="site"> | |
| <div class="wrap"> | |
| <h1>Pev Leaderboard</h1> | |
| <p class="lede">Accuracy, calibration and paired comparisons on Pev-Bench: seven families of short decision questions a personal agent with long-term memory has to answer (which option the user would pick, whether an action needs approval, whether a memory applies, whether an action relies on a forgotten fact, whether a fact may be shared, how urgently to notify, and where to route a request).</p> | |
| <ul class="links" aria-label="Project links"> | |
| <li><a href="https://github.com/EnvLoop/Pev/blob/main/docs/tech-report/pev.pdf">Paper</a></li> | |
| <li><a href="https://huggingface.co/EnvLoop/Pev-27B-LoRA">Model</a></li> | |
| <li><a href="https://huggingface.co/datasets/EnvLoop/Pev-Bench">Dataset</a></li> | |
| <li><a href="https://github.com/EnvLoop/Pev">Code</a></li> | |
| <li><a href="https://huggingface.co/collections/EnvLoop/pev-6abf32b6b1940477ad4c52c3">Collection</a></li> | |
| </ul> | |
| <p class="note">EnvLoop Research. The model (Pev-27B, a LoRA adapter on Qwen3.8-27B) and the dataset (Pev-Bench) are gated and released for non-commercial research only.</p> | |
| </div> | |
| </header> | |
| <nav class="toc" aria-label="Sections"> | |
| <ul> | |
| <li><a href="#test">TEST leaderboard</a></li> | |
| <li><a href="#paired">Paired comparisons</a></li> | |
| <li><a href="#families">Per family</a></li> | |
| <li><a href="#hidden">HIDDEN gate</a></li> | |
| <li><a href="#dev">DEV references</a></li> | |
| <li><a href="#caveats">Caveats</a></li> | |
| <li><a href="#figures">Figures</a></li> | |
| <li><a href="#cite">Citation</a></li> | |
| </ul> | |
| </nav> | |
| <main class="wrap"> | |
| <section id="test" aria-labelledby="test-h"> | |
| <h2 id="test-h">TEST leaderboard</h2> | |
| <p>The public TEST set has <strong id="t-states"></strong> states from 720 new users and <strong id="t-scorable"></strong> scorable questions. Half A (<span id="t-a"></span>) was rendered to text by gpt-6-astra and half B (<span id="t-b"></span>) by Claude Opus 5.5 under the same rendering contract. TEST was pseudonymized before any model was scored, so the released TEST is exactly the scored TEST. Every model was run once and all predictions were frozen before scoring.</p> | |
| <p>The primary metric is family-macro accuracy: the mean of the seven per-family accuracies, with questions whose soft label has no unique answer excluded. The 6-family columns are the pre-registered sensitivity analysis without <code>pick_option</code>. Brier and ECE are family-macro averages; automation coverage is the share of questions answered alone at the threshold fitted on VAL for a 5% error budget.</p> | |
| <div class="table-scroll"><table id="tbl-leader"></table></div> | |
| <p class="note">* gpt-6-astra and Jev return probabilities through their APIs; these are used as returned (no temperature fitted on VAL), so their Brier and ECE are uncalibrated and they have no automation threshold (n/a). Accuracy does not depend on calibration. The local models are scored from label-token log-probabilities with a temperature fitted on VAL.</p> | |
| </section> | |
| <section id="paired" aria-labelledby="paired-h"> | |
| <h2 id="paired-h">Paired comparisons</h2> | |
| <p>Pev-27B minus each other model, family-macro accuracy in points with a 95% confidence interval from a paired bootstrap over state clusters (10,000 resamples; <span id="p-clusters"></span> clusters on all of TEST). p-values are one-sided and Holm-adjusted over the five comparisons, separately for each column; 5 × 10<sup>−4</sup> is the smallest value attainable with 10,000 resamples and five comparisons.</p> | |
| <div class="controls"> | |
| <fieldset class="seg" id="seg-paired"> | |
| <legend>Families</legend> | |
| <input type="radio" name="pfam" id="pfam7" value="p7" checked><label for="pfam7">7 families (primary)</label> | |
| <input type="radio" name="pfam" id="pfam6" value="p6"><label for="pfam6">6 families (without pick_option)</label> | |
| </fieldset> | |
| </div> | |
| <div class="table-scroll"><table id="tbl-paired"></table></div> | |
| <p class="note">The sensitivity analysis without <code>pick_option</code> agrees with the primary result in every comparison and half (same sign, all significant after Holm correction).</p> | |
| </section> | |
| <section id="families" aria-labelledby="fam-h"> | |
| <h2 id="fam-h">Per family</h2> | |
| <div class="controls"> | |
| <div class="field"> | |
| <label for="famSel">Question family</label> | |
| <select id="famSel"></select> | |
| </div> | |
| </div> | |
| <p class="famdesc" id="famDesc"></p> | |
| <p class="note" id="famMeta"></p> | |
| <div class="table-scroll"><table id="tbl-fam"></table></div> | |
| <div class="callout" id="famPick" hidden><strong>pick_option is not solved.</strong> All six models score 0.413–0.480. A “priciest option” shortcut scores 0.347 on TEST against a chance level of 0.276 (+10.9 points over chance on half A, +2.9 on half B), and Pev-27B is not significantly ahead of gpt-6-astra or the base model on this family. No <code>pick_option</code> progress is claimed.</div> | |
| </section> | |
| <section id="hidden" aria-labelledby="hidden-h"> | |
| <h2 id="hidden-h">HIDDEN one-shot gate</h2> | |
| <p>Before TEST existed, the frozen Pev-27B adapter was evaluated exactly once on a sealed, pre-registered HIDDEN set (<span id="h-states"></span> states, <span id="h-scorable"></span> scorable of <span id="h-questions"></span> questions; 446 of the states use rendering styles never seen in training). The comparison is with the zero-shot base model. The run produced aggregate-only outputs and the sealed set was deleted afterwards. <strong>HIDDEN is not released.</strong> <span id="h-gate"></span></p> | |
| <div class="stats" id="h-stats"></div> | |
| <div class="table-scroll"><table id="tbl-hidden"></table></div> | |
| <div class="table-scroll"><table id="tbl-hidden-safety"></table></div> | |
| </section> | |
| <section id="dev" aria-labelledby="dev-h"> | |
| <h2 id="dev-h">DEV references</h2> | |
| <p>DEV (630 states, <span id="d-scorable"></span> scorable questions) was the model-selection set for Pev-27B, so its DEV score carries selection bias; HIDDEN and TEST confirm it. These numbers were computed on the DEV text before pseudonymization; the released DEV differs only in person names, contact details and user ids and was not re-scored, so re-running on it may give slightly different numbers.</p> | |
| <div class="table-scroll"><table id="tbl-dev"></table></div> | |
| <p class="note" id="d-paired"></p> | |
| </section> | |
| <section id="caveats" aria-labelledby="cav-h"> | |
| <h2 id="cav-h">Caveats</h2> | |
| <ul class="caveats"> | |
| <li><strong><code>pick_option</code> is essentially unsolved.</strong> It is the one family labelled by real behaviour (an item the user later rated 4 or higher). All six models score 0.41–0.48 on TEST; a “priciest option” shortcut scores 0.347 against a chance level of 0.276, so no <code>pick_option</code> progress is claimed. Pev-27B’s VAL threshold also over-automates this family (realized error 37% on accepted TEST questions against the 5% budget): a deployment should always ask the user here.</li> | |
| <li><strong>Safety false negatives.</strong> Pev-27B has no <code>forgotten_violation</code> false negatives on TEST, but gpt-6-astra is slightly lower on the other two safety families (0.7% / 0% / 0% against 0% / 1.3% / 2.0% for <code>forgotten_violation</code> / <code>needs_approval</code> / <code>share_ok</code>). Rates per model are below (all TEST users; about 150 questions at risk per family).</li> | |
| </ul> | |
| <div class="table-scroll"><table id="tbl-safety"></table></div> | |
| <ul class="caveats"> | |
| <li><strong>The text is LLM-rendered.</strong> Every state is rendered from structured facts under verbatim anchor checks, and memory facts are LLM-extracted. User histories are real public reviews, but approval rules, privacy levels, contacts and calendars are synthetic. Real user text and real rules will be messier.</li> | |
| <li><strong>Six of the seven families are rule-derived.</strong> High scores there show that the generator’s rules can be learned from rendered text, including buried and overridden evidence and an unseen renderer (TEST half B), not general judgement about approvals or privacy.</li> | |
| <li><strong>Renderer familiarity.</strong> gpt-6-astra rendered the training, DEV and TEST-A text. On TEST no home advantage is visible: gpt-6-astra scores 0.876 on half A and 0.871 on half B, Pev-27B 0.913 and 0.917.</li> | |
| <li><strong>Not a safety system.</strong> Approval and sharing predictions must be backed by hard rules in an agent; the model is a fast, calibrated advisor, not an enforcement layer.</li> | |
| </ul> | |
| </section> | |
| <section id="figures" aria-labelledby="fig-h"> | |
| <h2 id="fig-h">Figures</h2> | |
| <p class="note">From the technical report (CC-BY-4.0).</p> | |
| <div class="figs"> | |
| <figure> | |
| <div class="img"><img src="figures/fig6_test_macro.png" width="1251" height="655" loading="lazy" alt="Grouped bar chart of TEST family-macro accuracy for six models on all of TEST, half A and half B. Pev-27B is highest in each group at about 0.91 to 0.92, followed by gpt-6-astra at about 0.87, Kev-27B about 0.82, Jev about 0.78 to 0.80, the Qwen3.8-27B base about 0.76 and Qwen3.5-4B about 0.63 to 0.68."></div> | |
| <figcaption>TEST family-macro accuracy per model, overall and per renderer half (A: gpt-6-astra, B: Claude Opus 5.5).</figcaption> | |
| </figure> | |
| <figure> | |
| <div class="img"><img src="figures/fig7_test_deltas.png" width="1343" height="633" loading="lazy" alt="Dot plot with confidence intervals of Pev-27B minus each model on TEST, in points, for all of TEST, half A and half B. All intervals lie above zero: about 4 points versus gpt-6-astra, 9 versus Kev-27B, 12 to 13 versus Jev, 15 versus the Qwen3.8-27B base and 24 to 29 versus Qwen3.5-4B."></div> | |
| <figcaption>Pev-27B minus each model on TEST, family-macro accuracy with paired 95% confidence intervals, overall and per half.</figcaption> | |
| </figure> | |
| <figure> | |
| <div class="img"><img src="figures/fig1_hidden_accuracy.png" width="1290" height="695" loading="lazy" alt="Dumbbell chart of per-family accuracy on HIDDEN for the Qwen3.8-27B base and Pev-27B with chance marks. Gains in points: apply memory +15.8, forgotten violation +4.4, needs approval +15.3, share ok +16.7, route +8.1, notify level +42.2, pick option +3.3, macro +15.1."></div> | |
| <figcaption>Per-family accuracy on HIDDEN for the base model (orange) and Pev-27B (blue), with chance marks; labels give the change in points.</figcaption> | |
| </figure> | |
| <figure> | |
| <div class="img"><img src="figures/fig4_hidden_automation.png" width="1065" height="644" loading="lazy" alt="Scatter plot of automation coverage against realized error per family on HIDDEN at the VAL-fitted thresholds, with a horizontal line at the 5% error budget. Pev-27B covers all questions in six families with errors below the budget, but its pick option point sits at about 0.51 coverage and 0.45 realized error."></div> | |
| <figcaption>Coverage and realized error per family on HIDDEN at the VAL-fitted thresholds; the line is the 5% error budget.</figcaption> | |
| </figure> | |
| </div> | |
| </section> | |
| </main> | |
| <footer class="site" id="cite"> | |
| <div class="wrap"> | |
| <h2>Citation</h2> | |
| <div class="bibbar"> | |
| <button type="button" class="copy" id="copyBib" aria-describedby="copyStatus">Copy BibTeX</button> | |
| <span class="status" id="copyStatus" role="status" aria-live="polite"></span> | |
| </div> | |
| <pre class="bib" id="bib" tabindex="0">@techreport{envloop_pev, | |
| title = {Pev: A Calibrated Fast-Decision Model for Personal Agents}, | |
| author = {{EnvLoop Research}}, | |
| institution = {EnvLoop}, | |
| url = {https://github.com/EnvLoop/Pev} | |
| }</pre> | |
| <p>Contact and removal requests: <a href="mailto:research@envloop.ai">research@envloop.ai</a></p> | |
| <p class="note">The content of this page and its figures are licensed under CC-BY-4.0. The model and the dataset are gated and have their own non-commercial research licences; see their cards.</p> | |
| </div> | |
| </footer> | |
| <script> | |
| (function () { | |
| "use strict"; | |
| var DATA = {"models":[{"key":"pev","name":"Pev-27B","local":true,"ours":true},{"key":"astra","name":"gpt-6-astra","local":false,"ours":false},{"key":"kev","name":"Kev-27B","local":true,"ours":false},{"key":"jev","name":"Jev (jev-1.13.0)","local":false,"ours":false},{"key":"base","name":"Qwen3.8-27B (base)","local":true,"ours":false},{"key":"qwen4b","name":"Qwen3.5-4B","local":true,"ours":false}],"families":["apply_memory","forgotten_violation","needs_approval","share_ok","route","notify_level","pick_option"],"safetyFamilies":["forgotten_violation","needs_approval","share_ok"],"chance":{"apply_memory":0.5,"forgotten_violation":0.5,"needs_approval":0.5,"share_ok":0.5,"route":0.167,"notify_level":0.25,"pick_option":0.276},"test":{"n":{"all":{"questions":2506,"scorable":2156,"states":980},"A":{"questions":1252,"scorable":1073,"states":490},"B":{"questions":1254,"scorable":1083,"states":490}},"clusters":970,"nScored":{"all":{"apply_memory":302,"forgotten_violation":301,"needs_approval":298,"share_ok":303,"route":339,"notify_level":313,"pick_option":300},"A":{"apply_memory":145,"forgotten_violation":152,"needs_approval":147,"share_ok":150,"route":169,"notify_level":155,"pick_option":155},"B":{"apply_memory":157,"forgotten_violation":149,"needs_approval":151,"share_ok":153,"route":170,"notify_level":158,"pick_option":145}},"rows":{"pev":{"m7":{"all":0.915,"A":0.913,"B":0.917},"m6":{"all":0.987,"A":0.987,"B":0.987},"brier":{"all":0.125,"A":0.123,"B":0.128},"ece":{"all":0.065,"A":0.069,"B":0.065},"cov":{"all":0.94,"A":0.93,"B":0.94},"fam":{"apply_memory":{"all":1.0,"A":1.0,"B":1.0},"forgotten_violation":{"all":0.997,"A":1.0,"B":0.993},"needs_approval":{"all":0.993,"A":0.993,"B":0.993},"share_ok":{"all":0.99,"A":0.993,"B":0.987},"route":{"all":0.988,"A":0.982,"B":0.994},"notify_level":{"all":0.955,"A":0.955,"B":0.956},"pick_option":{"all":0.48,"A":0.465,"B":0.497}},"fn":{"all":{"forgotten_violation":0.0,"needs_approval":1.3,"share_ok":2.0},"A":{"forgotten_violation":0.0,"needs_approval":1.4,"share_ok":1.3},"B":{"forgotten_violation":0.0,"needs_approval":1.3,"share_ok":2.6}}},"astra":{"m7":{"all":0.873,"A":0.876,"B":0.871},"m6":{"all":0.945,"A":0.946,"B":0.944},"brier":{"all":0.18,"A":0.177,"B":0.184},"ece":{"all":0.074,"A":0.081,"B":0.079},"cov":{"all":null,"A":null,"B":null},"fam":{"apply_memory":{"all":0.977,"A":0.979,"B":0.975},"forgotten_violation":{"all":0.993,"A":1.0,"B":0.987},"needs_approval":{"all":0.993,"A":0.993,"B":0.993},"share_ok":{"all":1.0,"A":1.0,"B":1.0},"route":{"all":0.909,"A":0.905,"B":0.912},"notify_level":{"all":0.799,"A":0.8,"B":0.797},"pick_option":{"all":0.443,"A":0.452,"B":0.434}},"fn":{"all":{"forgotten_violation":0.7,"needs_approval":0.0,"share_ok":0.0},"A":{"forgotten_violation":0.0,"needs_approval":0.0,"share_ok":0.0},"B":{"forgotten_violation":1.3,"needs_approval":0.0,"share_ok":0.0}}},"kev":{"m7":{"all":0.823,"A":0.825,"B":0.821},"m6":{"all":0.888,"A":0.885,"B":0.891},"brier":{"all":0.218,"A":0.22,"B":0.217},"ece":{"all":0.074,"A":0.086,"B":0.09},"cov":{"all":0.71,"A":0.7,"B":0.71},"fam":{"apply_memory":{"all":0.977,"A":0.986,"B":0.968},"forgotten_violation":{"all":0.997,"A":1.0,"B":0.993},"needs_approval":{"all":0.893,"A":0.905,"B":0.881},"share_ok":{"all":0.977,"A":0.967,"B":0.987},"route":{"all":0.909,"A":0.882,"B":0.935},"notify_level":{"all":0.575,"A":0.568,"B":0.582},"pick_option":{"all":0.433,"A":0.465,"B":0.4}},"fn":{"all":{"forgotten_violation":0.0,"needs_approval":2.0,"share_ok":4.6},"A":{"forgotten_violation":0.0,"needs_approval":1.4,"share_ok":6.7},"B":{"forgotten_violation":0.0,"needs_approval":2.6,"share_ok":2.6}}},"jev":{"m7":{"all":0.788,"A":0.779,"B":0.797},"m6":{"all":0.851,"A":0.837,"B":0.864},"brier":{"all":0.266,"A":0.276,"B":0.257},"ece":{"all":0.123,"A":0.128,"B":0.129},"cov":{"all":null,"A":null,"B":null},"fam":{"apply_memory":{"all":0.957,"A":0.945,"B":0.968},"forgotten_violation":{"all":0.997,"A":1.0,"B":0.993},"needs_approval":{"all":0.883,"A":0.878,"B":0.887},"share_ok":{"all":0.875,"A":0.833,"B":0.915},"route":{"all":0.876,"A":0.864,"B":0.888},"notify_level":{"all":0.518,"A":0.503,"B":0.532},"pick_option":{"all":0.413,"A":0.432,"B":0.393}},"fn":{"all":{"forgotten_violation":0.0,"needs_approval":4.0,"share_ok":21.9},"A":{"forgotten_violation":0.0,"needs_approval":6.8,"share_ok":28.0},"B":{"forgotten_violation":0.0,"needs_approval":1.3,"share_ok":15.8}}},"base":{"m7":{"all":0.762,"A":0.759,"B":0.766},"m6":{"all":0.818,"A":0.811,"B":0.824},"brier":{"all":0.264,"A":0.268,"B":0.26},"ece":{"all":0.067,"A":0.077,"B":0.076},"cov":{"all":0.56,"A":0.55,"B":0.58},"fam":{"apply_memory":{"all":0.844,"A":0.807,"B":0.879},"forgotten_violation":{"all":0.967,"A":0.967,"B":0.966},"needs_approval":{"all":0.819,"A":0.81,"B":0.828},"share_ok":{"all":0.868,"A":0.86,"B":0.876},"route":{"all":0.917,"A":0.905,"B":0.929},"notify_level":{"all":0.492,"A":0.516,"B":0.468},"pick_option":{"all":0.43,"A":0.445,"B":0.414}},"fn":{"all":{"forgotten_violation":6.0,"needs_approval":4.7,"share_ok":4.6},"A":{"forgotten_violation":6.6,"needs_approval":5.5,"share_ok":6.7},"B":{"forgotten_violation":5.3,"needs_approval":3.9,"share_ok":2.6}}},"qwen4b":{"m7":{"all":0.652,"A":0.627,"B":0.677},"m6":{"all":0.686,"A":0.66,"B":0.711},"brier":{"all":0.389,"A":0.405,"B":0.374},"ece":{"all":0.074,"A":0.088,"B":0.09},"cov":{"all":0.11,"A":0.11,"B":0.1},"fam":{"apply_memory":{"all":0.758,"A":0.683,"B":0.828},"forgotten_violation":{"all":0.711,"A":0.671,"B":0.752},"needs_approval":{"all":0.628,"A":0.605,"B":0.649},"share_ok":{"all":0.653,"A":0.647,"B":0.66},"route":{"all":0.791,"A":0.811,"B":0.771},"notify_level":{"all":0.575,"A":0.542,"B":0.608},"pick_option":{"all":0.45,"A":0.432,"B":0.469}},"fn":{"all":{"forgotten_violation":56.3,"needs_approval":5.4,"share_ok":47.7},"A":{"forgotten_violation":65.8,"needs_approval":5.5,"share_ok":49.3},"B":{"forgotten_violation":46.7,"needs_approval":5.3,"share_ok":46.1}}}},"paired":{"astra":{"p7":{"all":{"d":4.1,"lo":2.8,"hi":5.5,"p":"5 × 10⁻⁴"},"A":{"d":3.7,"lo":1.9,"hi":5.5,"p":"5 × 10⁻⁴"},"B":{"d":4.6,"lo":2.6,"hi":6.6,"p":"5 × 10⁻⁴"}},"p6":{"all":{"d":4.2,"lo":3.1,"hi":5.3,"p":"5 × 10⁻⁴"},"A":{"d":4.1,"lo":2.6,"hi":5.6,"p":"5 × 10⁻⁴"},"B":{"d":4.3,"lo":2.8,"hi":5.9,"p":"5 × 10⁻⁴"}},"fam":{"apply_memory":{"all":{"d":2.3,"lo":0.7,"hi":4.2},"A":{"d":2.1,"lo":0.0,"hi":4.7},"B":{"d":2.5,"lo":0.6,"hi":5.3}},"forgotten_violation":{"all":{"d":0.3,"lo":0.0,"hi":1.1},"A":{"d":0.0,"lo":0.0,"hi":0.0},"B":{"d":0.7,"lo":0.0,"hi":2.2}},"needs_approval":{"all":{"d":0.0,"lo":-1.3,"hi":1.3},"A":{"d":0.0,"lo":-2.0,"hi":2.0},"B":{"d":0.0,"lo":-1.9,"hi":1.9}},"share_ok":{"all":{"d":-1.0,"lo":-2.3,"hi":0.0},"A":{"d":-0.7,"lo":-2.2,"hi":0.0},"B":{"d":-1.3,"lo":-3.4,"hi":0.0}},"route":{"all":{"d":8.0,"lo":5.0,"hi":11.2},"A":{"d":7.7,"lo":3.2,"hi":12.5},"B":{"d":8.2,"lo":4.4,"hi":12.6}},"notify_level":{"all":{"d":15.7,"lo":10.5,"hi":20.9},"A":{"d":15.5,"lo":8.2,"hi":23.1},"B":{"d":15.8,"lo":8.7,"hi":23.0}},"pick_option":{"all":{"d":3.7,"lo":-3.1,"hi":10.3},"A":{"d":1.3,"lo":-7.4,"hi":10.0},"B":{"d":6.2,"lo":-4.1,"hi":16.8}}}},"kev":{"p7":{"all":{"d":9.2,"lo":7.8,"hi":10.5,"p":"5 × 10⁻⁴"},"A":{"d":8.8,"lo":6.9,"hi":10.7,"p":"5 × 10⁻⁴"},"B":{"d":9.6,"lo":7.7,"hi":11.5,"p":"5 × 10⁻⁴"}},"p6":{"all":{"d":9.9,"lo":8.7,"hi":11.2,"p":"5 × 10⁻⁴"},"A":{"d":10.3,"lo":8.5,"hi":12.1,"p":"5 × 10⁻⁴"},"B":{"d":9.6,"lo":7.8,"hi":11.4,"p":"5 × 10⁻⁴"}},"fam":{"apply_memory":{"all":{"d":2.3,"lo":0.7,"hi":4.2},"A":{"d":1.4,"lo":0.0,"hi":3.6},"B":{"d":3.2,"lo":0.6,"hi":6.1}},"forgotten_violation":{"all":{"d":0.0,"lo":0.0,"hi":0.0},"A":{"d":0.0,"lo":0.0,"hi":0.0},"B":{"d":0.0,"lo":0.0,"hi":0.0}},"needs_approval":{"all":{"d":10.1,"lo":6.7,"hi":13.8},"A":{"d":8.8,"lo":4.6,"hi":13.7},"B":{"d":11.3,"lo":6.1,"hi":16.8}},"share_ok":{"all":{"d":1.3,"lo":0.0,"hi":3.0},"A":{"d":2.7,"lo":0.6,"hi":5.5},"B":{"d":0.0,"lo":-1.9,"hi":1.9}},"route":{"all":{"d":8.0,"lo":5.0,"hi":11.1},"A":{"d":10.1,"lo":5.5,"hi":15.1},"B":{"d":5.9,"lo":2.2,"hi":10.1}},"notify_level":{"all":{"d":38.0,"lo":32.4,"hi":43.8},"A":{"d":38.7,"lo":30.7,"hi":47.0},"B":{"d":37.3,"lo":29.4,"hi":45.3}},"pick_option":{"all":{"d":4.7,"lo":-1.4,"hi":10.5},"A":{"d":0.0,"lo":-8.3,"hi":8.1},"B":{"d":9.7,"lo":0.7,"hi":18.7}}}},"jev":{"p7":{"all":{"d":12.7,"lo":11.2,"hi":14.1,"p":"5 × 10⁻⁴"},"A":{"d":13.3,"lo":11.3,"hi":15.4,"p":"5 × 10⁻⁴"},"B":{"d":12.0,"lo":10.0,"hi":14.2,"p":"5 × 10⁻⁴"}},"p6":{"all":{"d":13.7,"lo":12.2,"hi":15.1,"p":"5 × 10⁻⁴"},"A":{"d":15.0,"lo":12.9,"hi":17.2,"p":"5 × 10⁻⁴"},"B":{"d":12.3,"lo":10.4,"hi":14.2,"p":"5 × 10⁻⁴"}},"fam":{"apply_memory":{"all":{"d":4.3,"lo":2.2,"hi":6.7},"A":{"d":5.5,"lo":2.1,"hi":9.6},"B":{"d":3.2,"lo":0.7,"hi":6.1}},"forgotten_violation":{"all":{"d":0.0,"lo":0.0,"hi":0.0},"A":{"d":0.0,"lo":0.0,"hi":0.0},"B":{"d":0.0,"lo":0.0,"hi":0.0}},"needs_approval":{"all":{"d":11.1,"lo":7.5,"hi":14.9},"A":{"d":11.6,"lo":6.6,"hi":16.9},"B":{"d":10.6,"lo":5.6,"hi":15.9}},"share_ok":{"all":{"d":11.6,"lo":8.0,"hi":15.3},"A":{"d":16.0,"lo":10.4,"hi":22.1},"B":{"d":7.2,"lo":3.4,"hi":11.6}},"route":{"all":{"d":11.2,"lo":7.8,"hi":14.8},"A":{"d":11.8,"lo":6.8,"hi":17.1},"B":{"d":10.6,"lo":5.9,"hi":15.7}},"notify_level":{"all":{"d":43.8,"lo":38.1,"hi":49.4},"A":{"d":45.2,"lo":36.9,"hi":53.6},"B":{"d":42.4,"lo":34.5,"hi":50.4}},"pick_option":{"all":{"d":6.7,"lo":1.0,"hi":12.5},"A":{"d":3.2,"lo":-4.1,"hi":10.3},"B":{"d":10.3,"lo":1.4,"hi":19.4}}}},"base":{"p7":{"all":{"d":15.2,"lo":13.6,"hi":16.8,"p":"5 × 10⁻⁴"},"A":{"d":15.4,"lo":13.0,"hi":17.9,"p":"5 × 10⁻⁴"},"B":{"d":15.1,"lo":12.9,"hi":17.3,"p":"5 × 10⁻⁴"}},"p6":{"all":{"d":16.9,"lo":15.4,"hi":18.6,"p":"5 × 10⁻⁴"},"A":{"d":17.6,"lo":15.2,"hi":20.0,"p":"5 × 10⁻⁴"},"B":{"d":16.3,"lo":14.1,"hi":18.6,"p":"5 × 10⁻⁴"}},"fam":{"apply_memory":{"all":{"d":15.6,"lo":11.6,"hi":19.9},"A":{"d":19.3,"lo":13.1,"hi":26.0},"B":{"d":12.1,"lo":7.3,"hi":17.4}},"forgotten_violation":{"all":{"d":3.0,"lo":1.3,"hi":5.1},"A":{"d":3.3,"lo":0.7,"hi":6.4},"B":{"d":2.7,"lo":0.6,"hi":5.6}},"needs_approval":{"all":{"d":17.4,"lo":13.1,"hi":22.0},"A":{"d":18.4,"lo":12.2,"hi":25.0},"B":{"d":16.6,"lo":10.6,"hi":22.9}},"share_ok":{"all":{"d":12.2,"lo":8.4,"hi":16.2},"A":{"d":13.3,"lo":8.2,"hi":19.2},"B":{"d":11.1,"lo":5.6,"hi":16.9}},"route":{"all":{"d":7.1,"lo":4.3,"hi":10.0},"A":{"d":7.7,"lo":3.5,"hi":12.2},"B":{"d":6.5,"lo":2.6,"hi":10.7}},"notify_level":{"all":{"d":46.3,"lo":40.5,"hi":52.2},"A":{"d":43.9,"lo":35.6,"hi":52.1},"B":{"d":48.7,"lo":40.3,"hi":56.9}},"pick_option":{"all":{"d":5.0,"lo":-1.2,"hi":11.2},"A":{"d":1.9,"lo":-6.9,"hi":10.5},"B":{"d":8.3,"lo":-0.6,"hi":17.2}}}},"qwen4b":{"p7":{"all":{"d":26.3,"lo":24.1,"hi":28.4,"p":"5 × 10⁻⁴"},"A":{"d":28.5,"lo":25.5,"hi":31.6,"p":"5 × 10⁻⁴"},"B":{"d":24.1,"lo":21.1,"hi":27.0,"p":"5 × 10⁻⁴"}},"p6":{"all":{"d":30.1,"lo":27.9,"hi":32.3,"p":"5 × 10⁻⁴"},"A":{"d":32.8,"lo":29.7,"hi":35.8,"p":"5 × 10⁻⁴"},"B":{"d":27.6,"lo":24.6,"hi":30.7,"p":"5 × 10⁻⁴"}},"fam":{"apply_memory":{"all":{"d":24.2,"lo":19.4,"hi":29.2},"A":{"d":31.7,"lo":24.2,"hi":39.4},"B":{"d":17.2,"lo":11.6,"hi":23.3}},"forgotten_violation":{"all":{"d":28.6,"lo":23.5,"hi":33.9},"A":{"d":32.9,"lo":25.5,"hi":40.6},"B":{"d":24.2,"lo":17.2,"hi":31.4}},"needs_approval":{"all":{"d":36.6,"lo":31.1,"hi":42.2},"A":{"d":38.8,"lo":30.6,"hi":47.0},"B":{"d":34.4,"lo":26.7,"hi":42.3}},"share_ok":{"all":{"d":33.7,"lo":28.4,"hi":39.1},"A":{"d":34.7,"lo":27.1,"hi":42.5},"B":{"d":32.7,"lo":25.3,"hi":40.3}},"route":{"all":{"d":19.8,"lo":15.5,"hi":24.1},"A":{"d":17.2,"lo":11.7,"hi":22.9},"B":{"d":22.4,"lo":16.0,"hi":29.0}},"notify_level":{"all":{"d":38.0,"lo":32.2,"hi":43.8},"A":{"d":41.3,"lo":33.1,"hi":49.4},"B":{"d":34.8,"lo":26.6,"hi":43.0}},"pick_option":{"all":{"d":3.0,"lo":-3.6,"hi":9.7},"A":{"d":3.2,"lo":-5.1,"hi":11.4},"B":{"d":2.8,"lo":-7.7,"hi":13.3}}}}},"holmP":["5 × 10⁻⁴"]},"hidden":{"states":960,"questions":2456,"scorable":2106,"clusters":934,"base":0.754,"cand":0.905,"d":15.1,"lo":13.5,"hi":16.8,"p":"1 × 10⁻⁴","mcnemar":{"candOnly":359,"baseOnly":48,"p":"5.7 × 10⁻⁶⁰"},"brier":[0.28,0.135],"ece":[0.068,0.067],"cov":[0.54,0.93],"accepted":[1161,1959],"fam":{"apply_memory":{"n":303,"base":0.842,"cand":1.0,"d":15.8,"lo":11.8,"hi":20.0,"covBase":0.68,"covCand":1.0,"errBase":0.044,"errCand":0.0},"forgotten_violation":{"n":293,"base":0.956,"cand":1.0,"d":4.4,"lo":2.2,"hi":6.9,"covBase":0.81,"covCand":1.0,"errBase":0.004,"errCand":0.0},"needs_approval":{"n":308,"base":0.834,"cand":0.987,"d":15.3,"lo":11.4,"hi":19.5,"covBase":0.71,"covCand":1.0,"errBase":0.106,"errCand":0.013},"share_ok":{"n":306,"base":0.807,"cand":0.974,"d":16.7,"lo":12.5,"hi":21.0,"covBase":0.62,"covCand":1.0,"errBase":0.026,"errCand":0.026},"route":{"n":322,"base":0.898,"cand":0.978,"d":8.1,"lo":5.1,"hi":11.2,"covBase":0.84,"covCand":1.0,"errBase":0.074,"errCand":0.022},"notify_level":{"n":275,"base":0.545,"cand":0.967,"d":42.2,"lo":35.7,"hi":48.3,"covBase":0.06,"covCand":1.0,"errBase":0.312,"errCand":0.033},"pick_option":{"n":299,"base":0.398,"cand":0.431,"d":3.3,"lo":-3.1,"hi":9.6,"covBase":0.09,"covCand":0.51,"errBase":0.269,"errCand":0.447}},"safety":{"forgotten_violation":{"base":8.8,"cand":0.0,"d":-8.8,"lo":-13.5,"hi":-4.7,"n":148},"needs_approval":{"base":1.3,"cand":0.0,"d":-1.3,"lo":-3.3,"hi":0.0,"n":153},"share_ok":{"base":11.0,"cand":3.9,"d":-7.1,"lo":-11.7,"hi":-2.6,"n":154}},"gatePassed":true,"checksPassed":7,"checksTotal":7},"dev":{"scorable":1409,"order":["pev","astra","kev","jev","base","qwen4b"],"rows":{"pev":{"m7":0.907,"brier":0.131,"ece":0.066,"fam":{"apply_memory":1.0,"forgotten_violation":1.0,"needs_approval":0.995,"share_ok":0.995,"route":0.99,"notify_level":0.912,"pick_option":0.457},"fn":{"forgotten_violation":0.0,"needs_approval":0.0,"share_ok":1.0}},"astra":{"m7":0.881,"brier":0.178,"ece":0.075,"fam":{"apply_memory":0.977,"forgotten_violation":0.995,"needs_approval":0.995,"share_ok":0.971,"route":0.954,"notify_level":0.83,"pick_option":0.442},"fn":{"forgotten_violation":1.0,"needs_approval":0.0,"share_ok":1.0}},"kev":{"m7":0.824,"brier":0.213,"ece":0.075,"fam":{"apply_memory":0.972,"forgotten_violation":1.0,"needs_approval":0.954,"share_ok":0.962,"route":0.909,"notify_level":0.577,"pick_option":0.396},"fn":{"forgotten_violation":0.0,"needs_approval":0.0,"share_ok":7.7}},"jev":{"m7":0.785,"brier":0.279,"ece":0.12,"fam":{"apply_memory":0.944,"forgotten_violation":0.995,"needs_approval":0.923,"share_ok":0.837,"route":0.868,"notify_level":0.552,"pick_option":0.376},"fn":{"forgotten_violation":0.0,"needs_approval":0.0,"share_ok":27.9}},"base":{"m7":0.766,"brier":0.261,"ece":0.074,"fam":{"apply_memory":0.85,"forgotten_violation":0.966,"needs_approval":0.892,"share_ok":0.847,"route":0.919,"notify_level":0.515,"pick_option":0.376},"fn":{"forgotten_violation":6.9,"needs_approval":2.1,"share_ok":4.8}},"qwen4b":{"m7":0.638,"brier":0.408,"ece":0.068,"fam":{"apply_memory":0.737,"forgotten_violation":0.737,"needs_approval":0.686,"share_ok":0.636,"route":0.731,"notify_level":0.598,"pick_option":0.34},"fn":{"forgotten_violation":52.0,"needs_approval":6.2,"share_ok":51.9}}},"paired":{"jev":{"d":12.2,"lo":10.4,"hi":14.0,"p":"< 10⁻⁴","clusters":625},"astra":{"d":2.7,"lo":1.1,"hi":4.2,"p":"7 × 10⁻⁴","clusters":625}}}}; | |
| var FAM_INFO = { | |
| apply_memory: { label: "apply_memory", type: "yes/no", desc: "Is a remembered fact relevant to the current request? Guards against over-personalisation." }, | |
| forgotten_violation: { label: "forgotten_violation", type: "yes/no", desc: "Would the candidate action rely on a fact the user asked the agent to forget?" }, | |
| needs_approval: { label: "needs_approval", type: "yes/no", desc: "Does the action need the user's approval under the spend threshold in force, the new-recipient rule and the irreversible-action rule?" }, | |
| share_ok: { label: "share_ok", type: "yes/no", desc: "May a fact be shared with this recipient, given the recipient's clearance and the fact's privacy level?" }, | |
| route: { label: "route", type: "choice (6 labels)", desc: "Which connected service should handle the request, or should the agent ask the user?" }, | |
| notify_level: { label: "notify_level", type: "score (4 levels)", desc: "How urgently to notify (silent, digest, notify, interrupt), with quiet hours, a meeting cap and VIP bypass." }, | |
| pick_option: { label: "pick_option", type: "choice", desc: "Which candidate item would the user pick? Labelled by real later behaviour (an item the user rated 4 or higher)." } | |
| }; | |
| function el(tag, attrs, children) { | |
| var e = document.createElement(tag); | |
| if (attrs) { | |
| for (var k in attrs) { | |
| if (!Object.prototype.hasOwnProperty.call(attrs, k)) continue; | |
| if (k === "text") e.textContent = attrs[k]; | |
| else if (k === "html") e.innerHTML = attrs[k]; | |
| else if (k === "cls") e.className = attrs[k]; | |
| else e.setAttribute(k, attrs[k]); | |
| } | |
| } | |
| (children || []).forEach(function (c) { if (c) e.appendChild(typeof c === "string" ? document.createTextNode(c) : c); }); | |
| return e; | |
| } | |
| function setText(id, t) { var n = document.getElementById(id); if (n) n.textContent = t; } | |
| function int(n) { return n.toLocaleString("en-US"); } | |
| var MINUS = "−"; | |
| function f3(x) { return x == null ? "n/a" : x.toFixed(3); } | |
| function f2(x) { return x == null ? "n/a" : x.toFixed(2); } | |
| function pt(x) { | |
| if (x > 0) return "+" + x.toFixed(1); | |
| if (x < 0) return MINUS + Math.abs(x).toFixed(1); | |
| return "0.0"; | |
| } | |
| function dci(o) { return pt(o.d) + " [" + pt(o.lo) + ", " + pt(o.hi) + "]"; } | |
| function pc(x) { return x.toFixed(1) + "%"; } | |
| var models = DATA.models; | |
| var byKey = {}; | |
| models.forEach(function (m) { byKey[m.key] = m; }); | |
| var T = DATA.test; | |
| function nameCell(m, extraTag) { | |
| var th = el("th", { scope: "row", cls: "l sticky" }, [m.name]); | |
| if (extraTag) th.appendChild(el("span", { cls: "tag", text: extraTag })); | |
| return th; | |
| } | |
| function headRow(cells) { | |
| var tr = el("tr"); | |
| cells.forEach(function (c) { | |
| var a = { scope: c.scope || "col", cls: c.cls || "num" }; | |
| if (c.colspan) a.colspan = c.colspan; | |
| if (c.rowspan) a.rowspan = c.rowspan; | |
| if (c.title) a.title = c.title; | |
| tr.appendChild(el("th", a, [c.t])); | |
| }); | |
| return tr; | |
| } | |
| function td(text, cls) { return el("td", { cls: "num " + (cls || "") }, [text]); } | |
| // ---------- header numbers ---------- | |
| setText("t-states", int(T.n.all.states)); | |
| setText("t-scorable", int(T.n.all.scorable)); | |
| setText("t-a", int(T.n.A.states) + " states, " + int(T.n.A.scorable) + " scorable"); | |
| setText("t-b", int(T.n.B.states) + " states, " + int(T.n.B.scorable) + " scorable"); | |
| setText("p-clusters", int(T.clusters)); | |
| // ---------- leaderboard ---------- | |
| (function () { | |
| var t = document.getElementById("tbl-leader"); | |
| t.appendChild(el("caption", { text: "TEST, sorted by family-macro accuracy over all users (7 families). A: gpt-6-astra-rendered half; B: Claude Opus 5.5-rendered half." })); | |
| var thead = el("thead"); | |
| thead.appendChild(headRow([ | |
| { t: "#", rowspan: 2, cls: "num" }, | |
| { t: "Model", rowspan: 2, cls: "l sticky" }, | |
| { t: "7 families (primary)", colspan: 3, cls: "grp", scope: "colgroup" }, | |
| { t: "6 families (without pick_option)", colspan: 3, cls: "grp", scope: "colgroup" }, | |
| { t: "Calibration and automation (7 families, all)", colspan: 3, cls: "grp", scope: "colgroup" } | |
| ])); | |
| thead.appendChild(headRow([ | |
| { t: "All", cls: "num bl" }, { t: "A half" }, { t: "B half" }, | |
| { t: "All", cls: "num bl" }, { t: "A half" }, { t: "B half" }, | |
| { t: "Brier ↓", cls: "num bl", title: "Family-macro Brier score; lower is better" }, | |
| { t: "ECE ↓", title: "Family-macro expected calibration error; lower is better" }, | |
| { t: "Coverage @5%", title: "Share of questions answered alone at the VAL threshold for a 5% error budget" } | |
| ])); | |
| t.appendChild(thead); | |
| var tb = el("tbody"); | |
| models.forEach(function (m, i) { | |
| var r = T.rows[m.key]; | |
| var tr = el("tr", { cls: m.ours ? "ours" : "" }); | |
| tr.appendChild(td(String(i + 1), "dim")); | |
| tr.appendChild(nameCell(m, m.local ? null : "API")); | |
| tr.appendChild(td(f3(r.m7.all), "bl")); | |
| tr.appendChild(td(f3(r.m7.A))); | |
| tr.appendChild(td(f3(r.m7.B))); | |
| tr.appendChild(td(f3(r.m6.all), "bl")); | |
| tr.appendChild(td(f3(r.m6.A))); | |
| tr.appendChild(td(f3(r.m6.B))); | |
| var star = m.local ? "" : "*"; | |
| tr.appendChild(td(f3(r.brier.all) + star, "bl")); | |
| tr.appendChild(td(f3(r.ece.all) + star)); | |
| tr.appendChild(td(f2(r.cov.all))); | |
| tb.appendChild(tr); | |
| }); | |
| t.appendChild(tb); | |
| })(); | |
| // ---------- paired ---------- | |
| function renderPaired(which) { | |
| var t = document.getElementById("tbl-paired"); | |
| t.textContent = ""; | |
| var label = which === "p7" ? "7 families (primary)" : "6 families, without pick_option (sensitivity)"; | |
| t.appendChild(el("caption", { text: "Pev-27B minus model, family-macro accuracy in points [95% CI]; " + label + "." })); | |
| var thead = el("thead"); | |
| thead.appendChild(headRow([ | |
| { t: "Pev-27B minus", cls: "l sticky" }, | |
| { t: "All users" }, { t: "A half (gpt-6-astra)" }, { t: "B half (Claude Opus 5.5)" }, | |
| { t: "Holm-adjusted p (all / A / B)" } | |
| ])); | |
| t.appendChild(thead); | |
| var tb = el("tbody"); | |
| models.forEach(function (m) { | |
| if (m.ours) return; | |
| var p = T.paired[m.key][which]; | |
| var tr = el("tr"); | |
| tr.appendChild(nameCell(m, null)); | |
| tr.appendChild(td(dci(p.all))); | |
| tr.appendChild(td(dci(p.A))); | |
| tr.appendChild(td(dci(p.B))); | |
| var ps = [p.all.p, p.A.p, p.B.p]; | |
| tr.appendChild(td(ps[0] === ps[1] && ps[1] === ps[2] ? ps[0] + " (each)" : ps.join(" / "))); | |
| tb.appendChild(tr); | |
| }); | |
| t.appendChild(tb); | |
| } | |
| renderPaired("p7"); | |
| Array.prototype.forEach.call(document.querySelectorAll('input[name="pfam"]'), function (r) { | |
| r.addEventListener("change", function () { if (r.checked) renderPaired(r.value); }); | |
| }); | |
| // ---------- per family ---------- | |
| var sel = document.getElementById("famSel"); | |
| DATA.families.forEach(function (f) { | |
| sel.appendChild(el("option", { value: f, text: f })); | |
| }); | |
| function chanceText(f) { | |
| var c = DATA.chance[f]; | |
| if (f === "route") return "1/6 (" + c.toFixed(3) + ")"; | |
| if (f === "pick_option") return c.toFixed(3) + " on TEST"; | |
| return c.toFixed(c === 0.5 ? 1 : 2); | |
| } | |
| function renderFam(f) { | |
| var info = FAM_INFO[f]; | |
| var d = document.getElementById("famDesc"); | |
| d.textContent = ""; | |
| d.appendChild(el("strong", { text: info.label })); | |
| d.appendChild(document.createTextNode(" (" + info.type + "). " + info.desc)); | |
| setText("famMeta", "Chance level: " + chanceText(f) + ". Scorable questions (same for every model): all " + | |
| int(T.nScored.all[f]) + ", A half " + int(T.nScored.A[f]) + ", B half " + int(T.nScored.B[f]) + "."); | |
| var t = document.getElementById("tbl-fam"); | |
| t.textContent = ""; | |
| t.appendChild(el("caption", { text: "TEST accuracy on " + f + ", sorted by accuracy over all users; last column: Pev-27B minus model on this family, points [95% CI], not adjusted for multiple comparisons." })); | |
| var thead = el("thead"); | |
| thead.appendChild(headRow([ | |
| { t: "Model", cls: "l sticky" }, { t: "All" }, { t: "A half" }, { t: "B half" }, | |
| { t: "Pev-27B minus model (all)", cls: "num bl" } | |
| ])); | |
| t.appendChild(thead); | |
| var tb = el("tbody"); | |
| var ms = models.slice().sort(function (a, b) { return T.rows[b.key].fam[f].all - T.rows[a.key].fam[f].all; }); | |
| ms.forEach(function (m) { | |
| var r = T.rows[m.key].fam[f]; | |
| var tr = el("tr", { cls: m.ours ? "ours" : "" }); | |
| tr.appendChild(nameCell(m, m.local ? null : "API")); | |
| tr.appendChild(td(f3(r.all))); | |
| tr.appendChild(td(f3(r.A))); | |
| tr.appendChild(td(f3(r.B))); | |
| tr.appendChild(td(m.ours ? "—" : dci(T.paired[m.key].fam[f].all), "bl")); | |
| tb.appendChild(tr); | |
| }); | |
| var c = DATA.chance[f]; | |
| var cr = el("tr", { cls: "chance" }); | |
| cr.appendChild(el("th", { scope: "row", cls: "l sticky" }, ["Chance"])); | |
| cr.appendChild(td(c.toFixed(3))); | |
| cr.appendChild(td("")); | |
| cr.appendChild(td("")); | |
| cr.appendChild(td("", "bl")); | |
| tb.appendChild(cr); | |
| t.appendChild(tb); | |
| document.getElementById("famPick").hidden = f !== "pick_option"; | |
| } | |
| sel.addEventListener("change", function () { renderFam(sel.value); }); | |
| renderFam(sel.value || DATA.families[0]); | |
| // ---------- HIDDEN ---------- | |
| (function () { | |
| var H = DATA.hidden; | |
| setText("h-states", int(H.states)); | |
| setText("h-scorable", int(H.scorable)); | |
| setText("h-questions", int(H.questions)); | |
| setText("h-gate", "All " + H.checksTotal + " pre-registered gate checks passed."); | |
| var stats = document.getElementById("h-stats"); | |
| function stat(k, v, s) { | |
| stats.appendChild(el("div", { cls: "stat" }, [el("div", { cls: "k", text: k }), el("div", { cls: "v", text: v }), el("div", { cls: "s", text: s })])); | |
| } | |
| stat("Family-macro accuracy", f3(H.base) + " → " + f3(H.cand), "Qwen3.8-27B (base) → Pev-27B"); | |
| stat("Difference", pt(H.d) + " points", "95% CI [" + pt(H.lo) + ", " + pt(H.hi) + "]; one-sided p = " + H.p + " (" + int(H.clusters) + " state clusters)"); | |
| stat("McNemar", H.mcnemar.candOnly + " vs " + H.mcnemar.baseOnly, "questions correct only for Pev-27B vs only for the base; exact p = " + H.mcnemar.p); | |
| stat("Automation coverage at 5% error", f2(H.cov[0]) + " → " + f2(H.cov[1]), "accepted questions " + int(H.accepted[0]) + " → " + int(H.accepted[1]) + "; Brier " + f3(H.brier[0]) + " → " + f3(H.brier[1])); | |
| var t = document.getElementById("tbl-hidden"); | |
| t.appendChild(el("caption", { text: "HIDDEN per family: Qwen3.8-27B (base) vs Pev-27B. Coverage and realized error at the VAL-fitted thresholds (5% error budget)." })); | |
| var thead = el("thead"); | |
| thead.appendChild(headRow([ | |
| { t: "Family", cls: "l sticky" }, { t: "n scored" }, { t: "Base" }, { t: "Pev-27B" }, | |
| { t: "Difference, points [95% CI]" }, { t: "Coverage base / ours", cls: "num bl" }, { t: "Realized error base / ours" } | |
| ])); | |
| t.appendChild(thead); | |
| var tb = el("tbody"); | |
| DATA.families.forEach(function (f) { | |
| var r = H.fam[f]; | |
| var tr = el("tr"); | |
| tr.appendChild(el("th", { scope: "row", cls: "l sticky" }, [f])); | |
| tr.appendChild(td(int(r.n))); | |
| tr.appendChild(td(f3(r.base))); | |
| tr.appendChild(td(f3(r.cand))); | |
| tr.appendChild(td(dci(r) + (r.lo <= 0 ? " n.s." : ""))); | |
| tr.appendChild(td(f2(r.covBase) + " / " + f2(r.covCand), "bl")); | |
| tr.appendChild(td(f3(r.errBase) + " / " + f3(r.errCand))); | |
| tb.appendChild(tr); | |
| }); | |
| var mr = el("tr", { cls: "ours" }); | |
| mr.appendChild(el("th", { scope: "row", cls: "l sticky" }, ["Macro (7 families)"])); | |
| mr.appendChild(td(int(H.scorable))); | |
| mr.appendChild(td(f3(H.base))); | |
| mr.appendChild(td(f3(H.cand))); | |
| mr.appendChild(td(dci(H))); | |
| mr.appendChild(td(f2(H.cov[0]) + " / " + f2(H.cov[1]), "bl")); | |
| mr.appendChild(td("")); | |
| tb.appendChild(mr); | |
| t.appendChild(tb); | |
| var s = document.getElementById("tbl-hidden-safety"); | |
| s.appendChild(el("caption", { text: "HIDDEN safety false-negative rates (questions at risk in brackets)." })); | |
| var sh = el("thead"); | |
| sh.appendChild(headRow([ | |
| { t: "Family", cls: "l sticky" }, { t: "Base" }, { t: "Pev-27B" }, { t: "Change, points [95% CI]" } | |
| ])); | |
| s.appendChild(sh); | |
| var sb = el("tbody"); | |
| DATA.safetyFamilies.forEach(function (f) { | |
| var r = H.safety[f]; | |
| var tr = el("tr"); | |
| tr.appendChild(el("th", { scope: "row", cls: "l sticky" }, [f + " (" + int(r.n) + ")"])); | |
| tr.appendChild(td(pc(r.base))); | |
| tr.appendChild(td(pc(r.cand))); | |
| tr.appendChild(td(dci(r))); | |
| sb.appendChild(tr); | |
| }); | |
| s.appendChild(sb); | |
| })(); | |
| // ---------- DEV ---------- | |
| (function () { | |
| var Dv = DATA.dev; | |
| setText("d-scorable", int(Dv.scorable)); | |
| var t = document.getElementById("tbl-dev"); | |
| t.appendChild(el("caption", { text: "DEV accuracy per family (pre-pseudonymization text), sorted by family-macro accuracy." })); | |
| var thead = el("thead"); | |
| var hc = [{ t: "Model", cls: "l sticky" }, { t: "Macro" }]; | |
| DATA.families.forEach(function (f, i) { hc.push({ t: f, cls: i === 0 ? "num bl" : "num" }); }); | |
| hc.push({ t: "Brier ↓", cls: "num bl" }, { t: "ECE ↓" }); | |
| thead.appendChild(headRow(hc)); | |
| t.appendChild(thead); | |
| var tb = el("tbody"); | |
| Dv.order.forEach(function (k) { | |
| var m = byKey[k], r = Dv.rows[k]; | |
| var tr = el("tr", { cls: m.ours ? "ours" : "" }); | |
| tr.appendChild(nameCell(m, m.local ? null : "API")); | |
| tr.appendChild(td(f3(r.m7))); | |
| DATA.families.forEach(function (f, i) { tr.appendChild(td(f3(r.fam[f]), i === 0 ? "bl" : "")); }); | |
| var star = m.local ? "" : "*"; | |
| tr.appendChild(td(f3(r.brier) + star, "bl")); | |
| tr.appendChild(td(f3(r.ece) + star)); | |
| tb.appendChild(tr); | |
| }); | |
| t.appendChild(tb); | |
| var a = Dv.paired.astra, j = Dv.paired.jev; | |
| setText("d-paired", "Paired over " + int(a.clusters) + " DEV state clusters: Pev-27B minus gpt-6-astra " + dci(a) + | |
| " points (one-sided p = " + a.p + "); minus Jev (jev-1.13.0) " + dci(j) + " points (one-sided p " + (j.p.charAt(0) === "<" ? j.p : "= " + j.p) + | |
| "). * Uncalibrated: API probabilities used as returned."); | |
| })(); | |
| // ---------- safety table (TEST) ---------- | |
| (function () { | |
| var t = document.getElementById("tbl-safety"); | |
| t.appendChild(el("caption", { text: "TEST safety false-negative rates, all users (lower is better)." })); | |
| var thead = el("thead"); | |
| var hc = [{ t: "Model", cls: "l sticky" }]; | |
| DATA.safetyFamilies.forEach(function (f) { hc.push({ t: f }); }); | |
| thead.appendChild(headRow(hc)); | |
| t.appendChild(thead); | |
| var tb = el("tbody"); | |
| models.forEach(function (m) { | |
| var r = T.rows[m.key].fn.all; | |
| var tr = el("tr", { cls: m.ours ? "ours" : "" }); | |
| tr.appendChild(nameCell(m, m.local ? null : "API")); | |
| DATA.safetyFamilies.forEach(function (f) { tr.appendChild(td(pc(r[f]))); }); | |
| tb.appendChild(tr); | |
| }); | |
| t.appendChild(tb); | |
| })(); | |
| // ---------- copy BibTeX ---------- | |
| (function () { | |
| var btn = document.getElementById("copyBib"); | |
| var status = document.getElementById("copyStatus"); | |
| var bib = document.getElementById("bib"); | |
| function done(ok) { | |
| status.textContent = ok ? "Copied." : "Copy failed; select the text and copy it manually."; | |
| setTimeout(function () { status.textContent = ""; }, 3000); | |
| } | |
| function fallback(text) { | |
| try { | |
| var ta = document.createElement("textarea"); | |
| ta.value = text; | |
| ta.setAttribute("readonly", ""); | |
| ta.style.position = "fixed"; | |
| ta.style.opacity = "0"; | |
| document.body.appendChild(ta); | |
| ta.select(); | |
| var ok = document.execCommand("copy"); | |
| document.body.removeChild(ta); | |
| done(ok); | |
| } catch (e) { done(false); } | |
| } | |
| btn.addEventListener("click", function () { | |
| var text = bib.textContent; | |
| try { | |
| if (navigator.clipboard && navigator.clipboard.writeText) { | |
| navigator.clipboard.writeText(text).then(function () { done(true); }, function () { fallback(text); }); | |
| } else { | |
| fallback(text); | |
| } | |
| } catch (e) { fallback(text); } | |
| }); | |
| })(); | |
| })(); | |
| </script> | |
| </body> | |
| </html> | |