File size: 5,142 Bytes
1d14f1a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | {
"model": "D1A-E2B-it v0.6 (experimental)",
"temperature": 1.414,
"method": "Sealed EVAL half: 487 hermes-agent PRs created after all training, held back before training, scored with d1a.eval.benchmark on the MLX 8-bit builds of E2B-it v0.6 (T 1.414) and E2B v0.6 (T 1.447) at their served temperatures, with the serving context; paired bootstrap over PRs (10,000 resamples, seed 0), 95% intervals. Its text is private. E4B v0.6 (served) is given for reference. Media: the playground's 14 photo, voice and video demo requests over HTTP, answers compared with E4B v0.6's baseline (28 answers).",
"bar_227": {
"pre_registered": "jonpol01/d1a#227, 2026-10-12, before any checkpoint existed",
"bar_b_apps": "FAIL on two lines: sealed-EVAL severity -4.3 [-7.8, -1.0] (needed >= -3); media agreement with E4B v0.6 21/28 (needed >= 25/28). Type passes (-0.2). The card-suite lines and the human read of the demo flips are not measured yet.",
"bar_a_trunk": "not measured yet (needs the card-suite gate)",
"release": "Published on the owner's decision of 2026-10-12 for an iOS proof of concept (an app that already loads Gemma 4 E2B-it), although the bar pre-registered for it (jonpol01/d1a#227, Bar B, 'good enough for apps') failed on two lines: sealed-EVAL severity -4.3 [-7.8, -1.0] against E2B v0.6 (needed >= -3) and media agreement with E4B v0.6 21/28 (needed >= 25/28). Experimental: E2B v0.6 stays E2B's model, and E4B v0.6 remains the PR labeler."
},
"fresh_prs_eval": {
"sev": {
"n": 487,
"acc": {
"E2B-it v0.6": 72.9,
"E2B v0.6": 77.2,
"E4B v0.6 (ref)": 79.1
},
"it_minus_e2b_v0.6": {
"delta": -4.3,
"ci95": [
-7.8,
-1.0
]
},
"recall": {
"P0": {
"E2B-it v0.6": "2/7",
"E2B v0.6": "0/7",
"E4B v0.6 (ref)": "5/7"
},
"P1": {
"E2B-it v0.6": "1/23",
"E2B v0.6": "0/23",
"E4B v0.6 (ref)": "8/23"
},
"P2": {
"E2B-it v0.6": "74/121",
"E2B v0.6": "77/121",
"E4B v0.6 (ref)": "66/121"
},
"P3": {
"E2B-it v0.6": "273/330",
"E2B v0.6": "295/330",
"E4B v0.6 (ref)": "303/330"
},
"P4": {
"E2B-it v0.6": "5/6",
"E2B v0.6": "4/6",
"E4B v0.6 (ref)": "3/6"
}
}
},
"type": {
"n": 487,
"acc": {
"E2B-it v0.6": 87.3,
"E2B v0.6": 87.5,
"E4B v0.6 (ref)": 90.6
},
"it_minus_e2b_v0.6": {
"delta": -0.2,
"ci95": [
-3.1,
2.7
]
},
"recall": {
"type/bug": {
"E2B-it v0.6": "217/251",
"E2B v0.6": "215/251",
"E4B v0.6 (ref)": "230/251"
},
"type/docs": {
"E2B-it v0.6": "8/11",
"E2B v0.6": "9/11",
"E4B v0.6 (ref)": "7/11"
},
"type/feature": {
"E2B-it v0.6": "118/132",
"E2B v0.6": "119/132",
"E4B v0.6 (ref)": "124/132"
},
"type/perf": {
"E2B-it v0.6": "1/2",
"E2B v0.6": "1/2",
"E4B v0.6 (ref)": "1/2"
},
"type/refactor": {
"E2B-it v0.6": "50/56",
"E2B v0.6": "50/56",
"E4B v0.6 (ref)": "50/56"
},
"type/security": {
"E2B-it v0.6": "11/14",
"E2B v0.6": "12/14",
"E4B v0.6 (ref)": "9/14"
},
"type/test": {
"E2B-it v0.6": "20/21",
"E2B v0.6": "20/21",
"E4B v0.6 (ref)": "20/21"
}
}
}
},
"card_suites": {
"status": "not measured yet",
"suites": [
"transfer-v4",
"hard-v1",
"devtools-v1",
"documents-v1",
"decision-v7",
"JGLUE"
]
},
"media": {
"lines": [
"media: 14/14 answered (HTTP 200, every question typed); latency p50 504 ms, max 3199 ms",
" differs photo | damaged_door | place: at a front door -> on a sidewalk",
" differs photo | intact_locker | damaged: true -> false",
" differs photo | intact_mailbox | damaged: true -> false",
" differs photo | intact_mailbox | place: in a delivery locker -> on a sidewalk",
" differs photo | damaged_crushed_truck | damaged: true -> false",
" differs voice | en_directions | urgent: false -> true",
" differs voice | ja_nothome | urgent: false -> true",
" differs video | damaged_wet | damaged: true -> false",
" differs video | damaged_crushed_truck | damaged: true -> false",
"vs E2B v0.6 (JohnP1/d1a-e2b-mlx-q8@v0.6): 19/28 answers the same",
" differs photo | damaged_door | place: at a front door -> on a sidewalk",
" differs photo | intact_mailbox | place: in a mailbox -> on a sidewalk",
" differs photo | damaged_crushed_truck | damaged: true -> false",
" differs voice | en_directions | urgent: false -> true",
" differs voice | ja_nothome | urgent: false -> true",
" differs video | damaged_wet | damaged: true -> false",
" differs video | damaged_crushed_truck | damaged: true -> false",
"vs E4B v0.6 (Mini baseline): 21/28 answers the same"
],
"agreement_with_e4b_v0.6": {
"E2B-it v0.6": "21/28",
"E2B v0.6": "25/28"
},
"answered": "14/14",
"direction": "each 'differs' line reads: reference answer -> E2B-it answer; the lines above a 'vs' line are the differences from that reference"
},
"quantized_builds": "the MLX 4-bit and Core ML builds' Mac checks are on their own cards"
}
|