{
 "model": "D1A-E2B-it v0.6 (experimental)",
 "temperature": 1.414,
 "method": "Sealed EVAL half: 487 hermes-agent PRs created after all training, held back before training, scored with d1a.eval.benchmark on the MLX 8-bit builds of E2B-it v0.6 (T 1.414) and E2B v0.6 (T 1.447) at their served temperatures, with the serving context; paired bootstrap over PRs (10,000 resamples, seed 0), 95% intervals. Its text is private. E4B v0.6 (served) is given for reference. Media: the playground's 14 photo, voice and video demo requests over HTTP, answers compared with E4B v0.6's baseline (28 answers).",
 "bar_227": {
  "pre_registered": "jonpol01/d1a#227, 2026-10-12, before any checkpoint existed",
  "bar_b_apps": "FAIL on two lines: sealed-EVAL severity -4.3 [-7.8, -1.0] (needed >= -3); media agreement with E4B v0.6 21/28 (needed >= 25/28). Type passes (-0.2). The card-suite lines and the human read of the demo flips are not measured yet.",
  "bar_a_trunk": "not measured yet (needs the card-suite gate)",
  "release": "Published on the owner's decision of 2026-10-12 for an iOS proof of concept (an app that already loads Gemma 4 E2B-it), although the bar pre-registered for it (jonpol01/d1a#227, Bar B, 'good enough for apps') failed on two lines: sealed-EVAL severity -4.3 [-7.8, -1.0] against E2B v0.6 (needed >= -3) and media agreement with E4B v0.6 21/28 (needed >= 25/28). Experimental: E2B v0.6 stays E2B's model, and E4B v0.6 remains the PR labeler."
 },
 "fresh_prs_eval": {
  "sev": {
   "n": 487,
   "acc": {
    "E2B-it v0.6": 72.9,
    "E2B v0.6": 77.2,
    "E4B v0.6 (ref)": 79.1
   },
   "it_minus_e2b_v0.6": {
    "delta": -4.3,
    "ci95": [
     -7.8,
     -1.0
    ]
   },
   "recall": {
    "P0": {
     "E2B-it v0.6": "2/7",
     "E2B v0.6": "0/7",
     "E4B v0.6 (ref)": "5/7"
    },
    "P1": {
     "E2B-it v0.6": "1/23",
     "E2B v0.6": "0/23",
     "E4B v0.6 (ref)": "8/23"
    },
    "P2": {
     "E2B-it v0.6": "74/121",
     "E2B v0.6": "77/121",
     "E4B v0.6 (ref)": "66/121"
    },
    "P3": {
     "E2B-it v0.6": "273/330",
     "E2B v0.6": "295/330",
     "E4B v0.6 (ref)": "303/330"
    },
    "P4": {
     "E2B-it v0.6": "5/6",
     "E2B v0.6": "4/6",
     "E4B v0.6 (ref)": "3/6"
    }
   }
  },
  "type": {
   "n": 487,
   "acc": {
    "E2B-it v0.6": 87.3,
    "E2B v0.6": 87.5,
    "E4B v0.6 (ref)": 90.6
   },
   "it_minus_e2b_v0.6": {
    "delta": -0.2,
    "ci95": [
     -3.1,
     2.7
    ]
   },
   "recall": {
    "type/bug": {
     "E2B-it v0.6": "217/251",
     "E2B v0.6": "215/251",
     "E4B v0.6 (ref)": "230/251"
    },
    "type/docs": {
     "E2B-it v0.6": "8/11",
     "E2B v0.6": "9/11",
     "E4B v0.6 (ref)": "7/11"
    },
    "type/feature": {
     "E2B-it v0.6": "118/132",
     "E2B v0.6": "119/132",
     "E4B v0.6 (ref)": "124/132"
    },
    "type/perf": {
     "E2B-it v0.6": "1/2",
     "E2B v0.6": "1/2",
     "E4B v0.6 (ref)": "1/2"
    },
    "type/refactor": {
     "E2B-it v0.6": "50/56",
     "E2B v0.6": "50/56",
     "E4B v0.6 (ref)": "50/56"
    },
    "type/security": {
     "E2B-it v0.6": "11/14",
     "E2B v0.6": "12/14",
     "E4B v0.6 (ref)": "9/14"
    },
    "type/test": {
     "E2B-it v0.6": "20/21",
     "E2B v0.6": "20/21",
     "E4B v0.6 (ref)": "20/21"
    }
   }
  }
 },
 "card_suites": {
  "status": "not measured yet",
  "suites": [
   "transfer-v4",
   "hard-v1",
   "devtools-v1",
   "documents-v1",
   "decision-v7",
   "JGLUE"
  ]
 },
 "media": {
  "lines": [
   "media: 14/14 answered (HTTP 200, every question typed); latency p50 504 ms, max 3199 ms",
   "  differs photo | damaged_door | place: at a front door -> on a sidewalk",
   "  differs photo | intact_locker | damaged: true -> false",
   "  differs photo | intact_mailbox | damaged: true -> false",
   "  differs photo | intact_mailbox | place: in a delivery locker -> on a sidewalk",
   "  differs photo | damaged_crushed_truck | damaged: true -> false",
   "  differs voice | en_directions | urgent: false -> true",
   "  differs voice | ja_nothome | urgent: false -> true",
   "  differs video | damaged_wet | damaged: true -> false",
   "  differs video | damaged_crushed_truck | damaged: true -> false",
   "vs E2B v0.6 (JohnP1/d1a-e2b-mlx-q8@v0.6): 19/28 answers the same",
   "  differs photo | damaged_door | place: at a front door -> on a sidewalk",
   "  differs photo | intact_mailbox | place: in a mailbox -> on a sidewalk",
   "  differs photo | damaged_crushed_truck | damaged: true -> false",
   "  differs voice | en_directions | urgent: false -> true",
   "  differs voice | ja_nothome | urgent: false -> true",
   "  differs video | damaged_wet | damaged: true -> false",
   "  differs video | damaged_crushed_truck | damaged: true -> false",
   "vs E4B v0.6 (Mini baseline): 21/28 answers the same"
  ],
  "agreement_with_e4b_v0.6": {
   "E2B-it v0.6": "21/28",
   "E2B v0.6": "25/28"
  },
  "answered": "14/14",
  "direction": "each 'differs' line reads: reference answer -> E2B-it answer; the lines above a 'vs' line are the differences from that reference"
 },
 "quantized_builds": "the MLX 4-bit and Core ML builds' Mac checks are on their own cards"
}