{
 "model": "D1A-E4B v0.6",
 "temperature": 1.954,
 "method": "d1a.eval.benchmark through scripts/quality_gate.py and scripts/decide.py on the MLX 8-bit builds of v0.5 (T 1.782) and v0.6 (T 1.954), held-out partitions; paired bootstrap over questions (or PRs), 95% intervals. The fresh-PR set is the sealed EVAL half of hermes-agent PRs created after all training, scored with the serving context; its text is private.",
 "verdict_vs_v0.5": {
  "decide": "VETO: V1",
  "pooled": {
   "delta": -0.2,
   "ci95": [
    -0.98,
    0.52
   ],
   "lines": 21
  },
  "release": "shipped as an explicit exception (see provenance.json)"
 },
 "sets": {
  "devtools-v1": {
   "n": 1074,
   "v0.5": {
    "acc": 69.8,
    "ece": 0.048
   },
   "v0.6": {
    "acc": 70.1,
    "ece": 0.037
   },
   "delta": 0.3,
   "ci95": [
    -1.4,
    1.9
   ],
   "fixes_breaks": [
    53,
    50
   ]
  },
  "documents-v1": {
   "n": 920,
   "v0.5": {
    "acc": 87.5,
    "ece": 0.042
   },
   "v0.6": {
    "acc": 89.5,
    "ece": 0.028
   },
   "delta": 2.0,
   "ci95": [
    0.1,
    3.7
   ],
   "fixes_breaks": [
    43,
    25
   ]
  },
  "external_semif-v1": {
   "n": 144,
   "v0.5": {
    "acc": 84.7,
    "ece": 0.11
   },
   "v0.6": {
    "acc": 83.3,
    "ece": 0.095
   },
   "delta": -1.4,
   "ci95": [
    -4.9,
    2.1
   ],
   "fixes_breaks": [
    3,
    5
   ]
  },
  "external_typesafe-v1": {
   "n": 102,
   "v0.5": {
    "acc": 74.5,
    "ece": 0.104
   },
   "v0.6": {
    "acc": 74.5,
    "ece": 0.129
   },
   "delta": 0.0,
   "ci95": [
    -6.3,
    7.1
   ],
   "fixes_breaks": [
    7,
    7
   ]
  },
  "external_wanli-v1": {
   "n": 256,
   "v0.5": {
    "acc": 72.7,
    "ece": 0.097
   },
   "v0.6": {
    "acc": 70.3,
    "ece": 0.101
   },
   "delta": -2.3,
   "ci95": [
    -5.5,
    0.8
   ],
   "fixes_breaks": [
    6,
    12
   ]
  },
  "external_wanli-v2": {
   "n": 1002,
   "v0.5": {
    "acc": 67.1,
    "ece": 0.123
   },
   "v0.6": {
    "acc": 67.1,
    "ece": 0.072
   },
   "delta": 0.0,
   "ci95": [
    -1.9,
    1.8
   ],
   "fixes_breaks": [
    47,
    47
   ]
  },
  "hard-v1": {
   "n": 1083,
   "v0.5": {
    "acc": 71.5,
    "ece": 0.045
   },
   "v0.6": {
    "acc": 68.8,
    "ece": 0.043
   },
   "delta": -2.7,
   "ci95": [
    -4.9,
    -0.5
   ],
   "fixes_breaks": [
    61,
    90
   ]
  },
  "ja-jglue_development": {
   "n": 1500,
   "v0.5": {
    "acc": 82.1,
    "ece": 0.026
   },
   "v0.6": {
    "acc": 81.7,
    "ece": 0.042
   },
   "delta": -0.3,
   "ci95": [
    -1.6,
    0.9
   ],
   "fixes_breaks": [
    42,
    47
   ]
  },
  "ja-jglue_test": {
   "n": 1500,
   "v0.5": {
    "acc": 80.2,
    "ece": 0.024
   },
   "v0.6": {
    "acc": 80.5,
    "ece": 0.039
   },
   "delta": 0.3,
   "ci95": [
    -0.9,
    1.5
   ],
   "fixes_breaks": [
    51,
    46
   ]
  },
  "night2_assertion": {
   "n": 1020,
   "v0.5": {
    "acc": 94.4,
    "ece": 0.051
   },
   "v0.6": {
    "acc": 94.9,
    "ece": 0.056
   },
   "delta": 0.5,
   "ci95": [
    -0.4,
    1.4
   ],
   "fixes_breaks": [
    12,
    7
   ]
  },
  "night2_dates": {
   "n": 900,
   "v0.5": {
    "acc": 92.7,
    "ece": 0.028
   },
   "v0.6": {
    "acc": 92.1,
    "ece": 0.053
   },
   "delta": -0.6,
   "ci95": [
    -2.2,
    1.1
   ],
   "fixes_breaks": [
    36,
    41
   ]
  },
  "night2_unknowable": {
   "n": 525,
   "v0.5": {
    "acc": 68.2,
    "ece": 0.062
   },
   "v0.6": {
    "acc": 67.8,
    "ece": 0.058
   },
   "delta": -0.4,
   "ci95": [
    -2.7,
    1.7
   ],
   "fixes_breaks": [
    30,
    32
   ]
  },
  "pr-labels_development": {
   "n": 2030,
   "v0.5": {
    "acc": 80.9,
    "ece": 0.02
   },
   "v0.6": {
    "acc": 79.7,
    "ece": 0.017
   },
   "delta": -1.3,
   "ci95": [
    -2.6,
    0.0
   ],
   "fixes_breaks": [
    80,
    106
   ]
  },
  "pr-labels_development-ja": {
   "n": 198,
   "v0.5": {
    "acc": 77.3,
    "ece": 0.064
   },
   "v0.6": {
    "acc": 76.8,
    "ece": 0.053
   },
   "delta": -0.5,
   "ci95": [
    -4.1,
    3.1
   ],
   "fixes_breaks": [
    6,
    7
   ]
  },
  "pr-labels_real17": {
   "n": 51,
   "v0.5": {
    "acc": 64.7,
    "ece": 0.136
   },
   "v0.6": {
    "acc": 68.6,
    "ece": 0.181
   },
   "delta": 3.9,
   "ci95": [
    0.0,
    9.8
   ],
   "fixes_breaks": [
    2,
    0
   ]
  },
  "pr-labels_test": {
   "n": 2045,
   "v0.5": {
    "acc": 81.3,
    "ece": 0.038
   },
   "v0.6": {
    "acc": 80.2,
    "ece": 0.03
   },
   "delta": -1.1,
   "ci95": [
    -2.5,
    0.4
   ],
   "fixes_breaks": [
    106,
    128
   ]
  },
  "pr-labels_test-ja": {
   "n": 193,
   "v0.5": {
    "acc": 76.2,
    "ece": 0.071
   },
   "v0.6": {
    "acc": 72.5,
    "ece": 0.043
   },
   "delta": -3.6,
   "ci95": [
    -8.1,
    0.5
   ],
   "fixes_breaks": [
    6,
    13
   ]
  },
  "routing_factory-development": {
   "n": 640,
   "v0.5": {
    "acc": 90.5,
    "ece": 0.13
   },
   "v0.6": {
    "acc": 93.9,
    "ece": 0.091
   },
   "delta": 3.4,
   "ci95": [
    1.4,
    5.6
   ],
   "fixes_breaks": [
    30,
    8
   ]
  },
  "routing_generic-development": {
   "n": 270,
   "v0.5": {
    "acc": 98.9,
    "ece": 0.029
   },
   "v0.6": {
    "acc": 98.5,
    "ece": 0.015
   },
   "delta": -0.4,
   "ci95": [
    -1.5,
    0.7
   ],
   "fixes_breaks": [
    1,
    2
   ]
  },
  "routing_handlabelled-45": {
   "n": 45,
   "v0.5": {
    "acc": 100.0,
    "ece": 0.032
   },
   "v0.6": {
    "acc": 97.8,
    "ece": 0.014
   },
   "delta": -2.2,
   "ci95": [
    -6.7,
    0.0
   ],
   "fixes_breaks": [
    0,
    1
   ]
  },
  "v4_transfer-v4": {
   "n": 656,
   "v0.5": {
    "acc": 71.6,
    "ece": 0.106
   },
   "v0.6": {
    "acc": 73.0,
    "ece": 0.055
   },
   "delta": 1.4,
   "ci95": [
    -1.2,
    4.0
   ],
   "fixes_breaks": [
    32,
    23
   ]
  },
  "v7_decision-v7": {
   "n": 1264,
   "v0.5": {
    "acc": 85.3,
    "ece": 0.03
   },
   "v0.6": {
    "acc": 83.9,
    "ece": 0.015
   },
   "delta": -1.4,
   "ci95": [
    -2.9,
    0.0
   ],
   "fixes_breaks": [
    31,
    49
   ]
  },
  "report-only:transfer-v2": {
   "n": 560,
   "v0.5": {
    "acc": 74.1,
    "ece": 0.074
   },
   "v0.6": {
    "acc": 73.8,
    "ece": 0.041
   },
   "delta": -0.4,
   "ci95": [
    -2.3,
    1.6
   ],
   "fixes_breaks": [
    17,
    19
   ],
   "report_only": true
  },
  "report-only:v9_transfer-v9": {
   "n": 1046,
   "v0.5": {
    "acc": 65.3,
    "ece": 0.083
   },
   "v0.6": {
    "acc": 65.9,
    "ece": 0.054
   },
   "delta": 0.6,
   "ci95": [
    -1.3,
    2.4
   ],
   "fixes_breaks": [
    49,
    43
   ],
   "report_only": true
  }
 },
 "pr_label_questions": {
  "pr-labels_test:type": {
   "n": 953,
   "v0.5": 89.4,
   "v0.6": 85.3,
   "delta": -4.1,
   "ci95": [
    -6.0,
    -2.3
   ]
  },
  "pr-labels_test:blast": {
   "n": 139,
   "v0.5": 57.6,
   "v0.6": 61.9,
   "delta": 4.3,
   "ci95": [
    -2.2,
    10.8
   ]
  },
  "pr-labels_test:sev": {
   "n": 953,
   "v0.5": 76.7,
   "v0.6": 77.9,
   "delta": 1.2,
   "ci95": [
    -1.3,
    3.6
   ]
  },
  "pr-labels_development:type": {
   "n": 946,
   "v0.5": 90.8,
   "v0.6": 90.7,
   "delta": -0.1,
   "ci95": [
    -1.8,
    1.5
   ]
  },
  "pr-labels_development:blast": {
   "n": 138,
   "v0.5": 58.7,
   "v0.6": 52.9,
   "delta": -5.8,
   "ci95": [
    -11.6,
    0.0
   ]
  },
  "pr-labels_development:sev": {
   "n": 946,
   "v0.5": 74.3,
   "v0.6": 72.5,
   "delta": -1.8,
   "ci95": [
    -3.9,
    0.3
   ]
  },
  "pr-labels_test-ja:type": {
   "n": 92,
   "v0.5": 83.7,
   "v0.6": 79.3,
   "delta": -4.3,
   "ci95": [
    -9.8,
    0.0
   ]
  },
  "pr-labels_test-ja:blast": {
   "n": 9,
   "v0.5": 44.4,
   "v0.6": 44.4,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "pr-labels_test-ja:sev": {
   "n": 92,
   "v0.5": 71.7,
   "v0.6": 68.5,
   "delta": -3.3,
   "ci95": [
    -10.9,
    4.3
   ]
  },
  "pr-labels_development-ja:type": {
   "n": 95,
   "v0.5": 87.4,
   "v0.6": 85.3,
   "delta": -2.1,
   "ci95": [
    -6.3,
    1.1
   ]
  },
  "pr-labels_development-ja:blast": {
   "n": 8,
   "v0.5": 62.5,
   "v0.6": 50.0,
   "delta": -12.5,
   "ci95": [
    -37.5,
    0.0
   ]
  },
  "pr-labels_development-ja:sev": {
   "n": 95,
   "v0.5": 68.4,
   "v0.6": 70.5,
   "delta": 2.1,
   "ci95": [
    -4.2,
    7.4
   ]
  },
  "pr-labels_real17:type": {
   "n": 17,
   "v0.5": 82.4,
   "v0.6": 82.4,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "pr-labels_real17:blast": {
   "n": 17,
   "v0.5": 58.8,
   "v0.6": 70.6,
   "delta": 11.8,
   "ci95": [
    0.0,
    29.4
   ]
  },
  "pr-labels_real17:sev": {
   "n": 17,
   "v0.5": 52.9,
   "v0.6": 52.9,
   "delta": 0.0,
   "ci95": [
    0.0,
    0.0
   ]
  },
  "pr-labels_test:sev+offsets": {
   "n": 953,
   "v0.5": 77.0,
   "v0.6": 76.6,
   "delta": -0.4,
   "ci95": [
    -2.4,
    1.8
   ]
  }
 },
 "fresh_prs_eval": {
  "sev": {
   "n": 487,
   "majority_floor": 0.678,
   "acc": {
    "v0.4": 0.774,
    "v0.5": 0.747,
    "v0.6": 0.791
   },
   "v0.6_vs_v0.5": {
    "delta": 4.3,
    "ci95": [
     0.8,
     7.8
    ],
    "fixes": 52,
    "breaks": 31,
    "sign_p": 0.028
   },
   "v0.6_vs_v0.4": {
    "delta": 1.6,
    "ci95": [
     -0.8,
     4.1
    ]
   },
   "recall": {
    "v0.4": {
     "P0": 0.29,
     "P1": 0.35,
     "P2": 0.58,
     "P3": 0.89,
     "P4": 0.5
    },
    "v0.5": {
     "P0": 0.29,
     "P1": 0.22,
     "P2": 0.79,
     "P3": 0.78,
     "P4": 0.5
    },
    "v0.6": {
     "P0": 0.71,
     "P1": 0.35,
     "P2": 0.55,
     "P3": 0.92,
     "P4": 0.5
    }
   }
  },
  "type": {
   "n": 487,
   "majority_floor": 0.515,
   "acc": {
    "v0.4": 0.864,
    "v0.5": 0.867,
    "v0.6": 0.906
   },
   "v0.6_vs_v0.5": {
    "delta": 3.9,
    "ci95": [
     1.2,
     6.6
    ],
    "fixes": 31,
    "breaks": 12,
    "sign_p": 0.005
   },
   "v0.6_vs_v0.4": {
    "delta": 4.1,
    "ci95": [
     1.4,
     7.0
    ]
   }
  }
 }
}