| { |
| "model": "D1A-E2B-it v0.6 (experimental)", |
| "temperature": 1.414, |
| "method": "Sealed EVAL half: 487 hermes-agent PRs created after all training, held back before training, scored with d1a.eval.benchmark on the MLX 8-bit builds of E2B-it v0.6 (T 1.414) and E2B v0.6 (T 1.447) at their served temperatures, with the serving context; paired bootstrap over PRs (10,000 resamples, seed 0), 95% intervals. Its text is private. E4B v0.6 (served) is given for reference. Media: the playground's 14 photo, voice and video demo requests over HTTP, answers compared with E4B v0.6's baseline (28 answers).", |
| "bar_227": { |
| "pre_registered": "jonpol01/d1a#227, 2026-10-12, before any checkpoint existed", |
| "bar_b_apps": "FAIL on two lines: sealed-EVAL severity -4.3 [-7.8, -1.0] (needed >= -3); media agreement with E4B v0.6 21/28 (needed >= 25/28). Type passes (-0.2). The card-suite lines and the human read of the demo flips are not measured yet.", |
| "bar_a_trunk": "not measured yet (needs the card-suite gate)", |
| "release": "Published on the owner's decision of 2026-10-12 for an iOS proof of concept (an app that already loads Gemma 4 E2B-it), although the bar pre-registered for it (jonpol01/d1a#227, Bar B, 'good enough for apps') failed on two lines: sealed-EVAL severity -4.3 [-7.8, -1.0] against E2B v0.6 (needed >= -3) and media agreement with E4B v0.6 21/28 (needed >= 25/28). Experimental: E2B v0.6 stays E2B's model, and E4B v0.6 remains the PR labeler." |
| }, |
| "fresh_prs_eval": { |
| "sev": { |
| "n": 487, |
| "acc": { |
| "E2B-it v0.6": 72.9, |
| "E2B v0.6": 77.2, |
| "E4B v0.6 (ref)": 79.1 |
| }, |
| "it_minus_e2b_v0.6": { |
| "delta": -4.3, |
| "ci95": [ |
| -7.8, |
| -1.0 |
| ] |
| }, |
| "recall": { |
| "P0": { |
| "E2B-it v0.6": "2/7", |
| "E2B v0.6": "0/7", |
| "E4B v0.6 (ref)": "5/7" |
| }, |
| "P1": { |
| "E2B-it v0.6": "1/23", |
| "E2B v0.6": "0/23", |
| "E4B v0.6 (ref)": "8/23" |
| }, |
| "P2": { |
| "E2B-it v0.6": "74/121", |
| "E2B v0.6": "77/121", |
| "E4B v0.6 (ref)": "66/121" |
| }, |
| "P3": { |
| "E2B-it v0.6": "273/330", |
| "E2B v0.6": "295/330", |
| "E4B v0.6 (ref)": "303/330" |
| }, |
| "P4": { |
| "E2B-it v0.6": "5/6", |
| "E2B v0.6": "4/6", |
| "E4B v0.6 (ref)": "3/6" |
| } |
| } |
| }, |
| "type": { |
| "n": 487, |
| "acc": { |
| "E2B-it v0.6": 87.3, |
| "E2B v0.6": 87.5, |
| "E4B v0.6 (ref)": 90.6 |
| }, |
| "it_minus_e2b_v0.6": { |
| "delta": -0.2, |
| "ci95": [ |
| -3.1, |
| 2.7 |
| ] |
| }, |
| "recall": { |
| "type/bug": { |
| "E2B-it v0.6": "217/251", |
| "E2B v0.6": "215/251", |
| "E4B v0.6 (ref)": "230/251" |
| }, |
| "type/docs": { |
| "E2B-it v0.6": "8/11", |
| "E2B v0.6": "9/11", |
| "E4B v0.6 (ref)": "7/11" |
| }, |
| "type/feature": { |
| "E2B-it v0.6": "118/132", |
| "E2B v0.6": "119/132", |
| "E4B v0.6 (ref)": "124/132" |
| }, |
| "type/perf": { |
| "E2B-it v0.6": "1/2", |
| "E2B v0.6": "1/2", |
| "E4B v0.6 (ref)": "1/2" |
| }, |
| "type/refactor": { |
| "E2B-it v0.6": "50/56", |
| "E2B v0.6": "50/56", |
| "E4B v0.6 (ref)": "50/56" |
| }, |
| "type/security": { |
| "E2B-it v0.6": "11/14", |
| "E2B v0.6": "12/14", |
| "E4B v0.6 (ref)": "9/14" |
| }, |
| "type/test": { |
| "E2B-it v0.6": "20/21", |
| "E2B v0.6": "20/21", |
| "E4B v0.6 (ref)": "20/21" |
| } |
| } |
| } |
| }, |
| "card_suites": { |
| "status": "not measured yet", |
| "suites": [ |
| "transfer-v4", |
| "hard-v1", |
| "devtools-v1", |
| "documents-v1", |
| "decision-v7", |
| "JGLUE" |
| ] |
| }, |
| "media": { |
| "lines": [ |
| "media: 14/14 answered (HTTP 200, every question typed); latency p50 504 ms, max 3199 ms", |
| " differs photo | damaged_door | place: at a front door -> on a sidewalk", |
| " differs photo | intact_locker | damaged: true -> false", |
| " differs photo | intact_mailbox | damaged: true -> false", |
| " differs photo | intact_mailbox | place: in a delivery locker -> on a sidewalk", |
| " differs photo | damaged_crushed_truck | damaged: true -> false", |
| " differs voice | en_directions | urgent: false -> true", |
| " differs voice | ja_nothome | urgent: false -> true", |
| " differs video | damaged_wet | damaged: true -> false", |
| " differs video | damaged_crushed_truck | damaged: true -> false", |
| "vs E2B v0.6 (JohnP1/d1a-e2b-mlx-q8@v0.6): 19/28 answers the same", |
| " differs photo | damaged_door | place: at a front door -> on a sidewalk", |
| " differs photo | intact_mailbox | place: in a mailbox -> on a sidewalk", |
| " differs photo | damaged_crushed_truck | damaged: true -> false", |
| " differs voice | en_directions | urgent: false -> true", |
| " differs voice | ja_nothome | urgent: false -> true", |
| " differs video | damaged_wet | damaged: true -> false", |
| " differs video | damaged_crushed_truck | damaged: true -> false", |
| "vs E4B v0.6 (Mini baseline): 21/28 answers the same" |
| ], |
| "agreement_with_e4b_v0.6": { |
| "E2B-it v0.6": "21/28", |
| "E2B v0.6": "25/28" |
| }, |
| "answered": "14/14", |
| "direction": "each 'differs' line reads: reference answer -> E2B-it answer; the lines above a 'vs' line are the differences from that reference" |
| }, |
| "quantized_builds": "the MLX 4-bit and Core ML builds' Mac checks are on their own cards" |
| } |
|
|
