| { |
| "model": "D1A-E4B v0.6", |
| "temperature": 1.954, |
| "method": "d1a.eval.benchmark through scripts/quality_gate.py and scripts/decide.py on the MLX 8-bit builds of v0.5 (T 1.782) and v0.6 (T 1.954), held-out partitions; paired bootstrap over questions (or PRs), 95% intervals. The fresh-PR set is the sealed EVAL half of hermes-agent PRs created after all training, scored with the serving context; its text is private.", |
| "verdict_vs_v0.5": { |
| "decide": "VETO: V1", |
| "pooled": { |
| "delta": -0.2, |
| "ci95": [ |
| -0.98, |
| 0.52 |
| ], |
| "lines": 21 |
| }, |
| "release": "shipped as an explicit exception (see provenance.json)" |
| }, |
| "sets": { |
| "devtools-v1": { |
| "n": 1074, |
| "v0.5": { |
| "acc": 69.8, |
| "ece": 0.048 |
| }, |
| "v0.6": { |
| "acc": 70.1, |
| "ece": 0.037 |
| }, |
| "delta": 0.3, |
| "ci95": [ |
| -1.4, |
| 1.9 |
| ], |
| "fixes_breaks": [ |
| 53, |
| 50 |
| ] |
| }, |
| "documents-v1": { |
| "n": 920, |
| "v0.5": { |
| "acc": 87.5, |
| "ece": 0.042 |
| }, |
| "v0.6": { |
| "acc": 89.5, |
| "ece": 0.028 |
| }, |
| "delta": 2.0, |
| "ci95": [ |
| 0.1, |
| 3.7 |
| ], |
| "fixes_breaks": [ |
| 43, |
| 25 |
| ] |
| }, |
| "external_semif-v1": { |
| "n": 144, |
| "v0.5": { |
| "acc": 84.7, |
| "ece": 0.11 |
| }, |
| "v0.6": { |
| "acc": 83.3, |
| "ece": 0.095 |
| }, |
| "delta": -1.4, |
| "ci95": [ |
| -4.9, |
| 2.1 |
| ], |
| "fixes_breaks": [ |
| 3, |
| 5 |
| ] |
| }, |
| "external_typesafe-v1": { |
| "n": 102, |
| "v0.5": { |
| "acc": 74.5, |
| "ece": 0.104 |
| }, |
| "v0.6": { |
| "acc": 74.5, |
| "ece": 0.129 |
| }, |
| "delta": 0.0, |
| "ci95": [ |
| -6.3, |
| 7.1 |
| ], |
| "fixes_breaks": [ |
| 7, |
| 7 |
| ] |
| }, |
| "external_wanli-v1": { |
| "n": 256, |
| "v0.5": { |
| "acc": 72.7, |
| "ece": 0.097 |
| }, |
| "v0.6": { |
| "acc": 70.3, |
| "ece": 0.101 |
| }, |
| "delta": -2.3, |
| "ci95": [ |
| -5.5, |
| 0.8 |
| ], |
| "fixes_breaks": [ |
| 6, |
| 12 |
| ] |
| }, |
| "external_wanli-v2": { |
| "n": 1002, |
| "v0.5": { |
| "acc": 67.1, |
| "ece": 0.123 |
| }, |
| "v0.6": { |
| "acc": 67.1, |
| "ece": 0.072 |
| }, |
| "delta": 0.0, |
| "ci95": [ |
| -1.9, |
| 1.8 |
| ], |
| "fixes_breaks": [ |
| 47, |
| 47 |
| ] |
| }, |
| "hard-v1": { |
| "n": 1083, |
| "v0.5": { |
| "acc": 71.5, |
| "ece": 0.045 |
| }, |
| "v0.6": { |
| "acc": 68.8, |
| "ece": 0.043 |
| }, |
| "delta": -2.7, |
| "ci95": [ |
| -4.9, |
| -0.5 |
| ], |
| "fixes_breaks": [ |
| 61, |
| 90 |
| ] |
| }, |
| "ja-jglue_development": { |
| "n": 1500, |
| "v0.5": { |
| "acc": 82.1, |
| "ece": 0.026 |
| }, |
| "v0.6": { |
| "acc": 81.7, |
| "ece": 0.042 |
| }, |
| "delta": -0.3, |
| "ci95": [ |
| -1.6, |
| 0.9 |
| ], |
| "fixes_breaks": [ |
| 42, |
| 47 |
| ] |
| }, |
| "ja-jglue_test": { |
| "n": 1500, |
| "v0.5": { |
| "acc": 80.2, |
| "ece": 0.024 |
| }, |
| "v0.6": { |
| "acc": 80.5, |
| "ece": 0.039 |
| }, |
| "delta": 0.3, |
| "ci95": [ |
| -0.9, |
| 1.5 |
| ], |
| "fixes_breaks": [ |
| 51, |
| 46 |
| ] |
| }, |
| "night2_assertion": { |
| "n": 1020, |
| "v0.5": { |
| "acc": 94.4, |
| "ece": 0.051 |
| }, |
| "v0.6": { |
| "acc": 94.9, |
| "ece": 0.056 |
| }, |
| "delta": 0.5, |
| "ci95": [ |
| -0.4, |
| 1.4 |
| ], |
| "fixes_breaks": [ |
| 12, |
| 7 |
| ] |
| }, |
| "night2_dates": { |
| "n": 900, |
| "v0.5": { |
| "acc": 92.7, |
| "ece": 0.028 |
| }, |
| "v0.6": { |
| "acc": 92.1, |
| "ece": 0.053 |
| }, |
| "delta": -0.6, |
| "ci95": [ |
| -2.2, |
| 1.1 |
| ], |
| "fixes_breaks": [ |
| 36, |
| 41 |
| ] |
| }, |
| "night2_unknowable": { |
| "n": 525, |
| "v0.5": { |
| "acc": 68.2, |
| "ece": 0.062 |
| }, |
| "v0.6": { |
| "acc": 67.8, |
| "ece": 0.058 |
| }, |
| "delta": -0.4, |
| "ci95": [ |
| -2.7, |
| 1.7 |
| ], |
| "fixes_breaks": [ |
| 30, |
| 32 |
| ] |
| }, |
| "pr-labels_development": { |
| "n": 2030, |
| "v0.5": { |
| "acc": 80.9, |
| "ece": 0.02 |
| }, |
| "v0.6": { |
| "acc": 79.7, |
| "ece": 0.017 |
| }, |
| "delta": -1.3, |
| "ci95": [ |
| -2.6, |
| 0.0 |
| ], |
| "fixes_breaks": [ |
| 80, |
| 106 |
| ] |
| }, |
| "pr-labels_development-ja": { |
| "n": 198, |
| "v0.5": { |
| "acc": 77.3, |
| "ece": 0.064 |
| }, |
| "v0.6": { |
| "acc": 76.8, |
| "ece": 0.053 |
| }, |
| "delta": -0.5, |
| "ci95": [ |
| -4.1, |
| 3.1 |
| ], |
| "fixes_breaks": [ |
| 6, |
| 7 |
| ] |
| }, |
| "pr-labels_real17": { |
| "n": 51, |
| "v0.5": { |
| "acc": 64.7, |
| "ece": 0.136 |
| }, |
| "v0.6": { |
| "acc": 68.6, |
| "ece": 0.181 |
| }, |
| "delta": 3.9, |
| "ci95": [ |
| 0.0, |
| 9.8 |
| ], |
| "fixes_breaks": [ |
| 2, |
| 0 |
| ] |
| }, |
| "pr-labels_test": { |
| "n": 2045, |
| "v0.5": { |
| "acc": 81.3, |
| "ece": 0.038 |
| }, |
| "v0.6": { |
| "acc": 80.2, |
| "ece": 0.03 |
| }, |
| "delta": -1.1, |
| "ci95": [ |
| -2.5, |
| 0.4 |
| ], |
| "fixes_breaks": [ |
| 106, |
| 128 |
| ] |
| }, |
| "pr-labels_test-ja": { |
| "n": 193, |
| "v0.5": { |
| "acc": 76.2, |
| "ece": 0.071 |
| }, |
| "v0.6": { |
| "acc": 72.5, |
| "ece": 0.043 |
| }, |
| "delta": -3.6, |
| "ci95": [ |
| -8.1, |
| 0.5 |
| ], |
| "fixes_breaks": [ |
| 6, |
| 13 |
| ] |
| }, |
| "routing_factory-development": { |
| "n": 640, |
| "v0.5": { |
| "acc": 90.5, |
| "ece": 0.13 |
| }, |
| "v0.6": { |
| "acc": 93.9, |
| "ece": 0.091 |
| }, |
| "delta": 3.4, |
| "ci95": [ |
| 1.4, |
| 5.6 |
| ], |
| "fixes_breaks": [ |
| 30, |
| 8 |
| ] |
| }, |
| "routing_generic-development": { |
| "n": 270, |
| "v0.5": { |
| "acc": 98.9, |
| "ece": 0.029 |
| }, |
| "v0.6": { |
| "acc": 98.5, |
| "ece": 0.015 |
| }, |
| "delta": -0.4, |
| "ci95": [ |
| -1.5, |
| 0.7 |
| ], |
| "fixes_breaks": [ |
| 1, |
| 2 |
| ] |
| }, |
| "routing_handlabelled-45": { |
| "n": 45, |
| "v0.5": { |
| "acc": 100.0, |
| "ece": 0.032 |
| }, |
| "v0.6": { |
| "acc": 97.8, |
| "ece": 0.014 |
| }, |
| "delta": -2.2, |
| "ci95": [ |
| -6.7, |
| 0.0 |
| ], |
| "fixes_breaks": [ |
| 0, |
| 1 |
| ] |
| }, |
| "v4_transfer-v4": { |
| "n": 656, |
| "v0.5": { |
| "acc": 71.6, |
| "ece": 0.106 |
| }, |
| "v0.6": { |
| "acc": 73.0, |
| "ece": 0.055 |
| }, |
| "delta": 1.4, |
| "ci95": [ |
| -1.2, |
| 4.0 |
| ], |
| "fixes_breaks": [ |
| 32, |
| 23 |
| ] |
| }, |
| "v7_decision-v7": { |
| "n": 1264, |
| "v0.5": { |
| "acc": 85.3, |
| "ece": 0.03 |
| }, |
| "v0.6": { |
| "acc": 83.9, |
| "ece": 0.015 |
| }, |
| "delta": -1.4, |
| "ci95": [ |
| -2.9, |
| 0.0 |
| ], |
| "fixes_breaks": [ |
| 31, |
| 49 |
| ] |
| }, |
| "report-only:transfer-v2": { |
| "n": 560, |
| "v0.5": { |
| "acc": 74.1, |
| "ece": 0.074 |
| }, |
| "v0.6": { |
| "acc": 73.8, |
| "ece": 0.041 |
| }, |
| "delta": -0.4, |
| "ci95": [ |
| -2.3, |
| 1.6 |
| ], |
| "fixes_breaks": [ |
| 17, |
| 19 |
| ], |
| "report_only": true |
| }, |
| "report-only:v9_transfer-v9": { |
| "n": 1046, |
| "v0.5": { |
| "acc": 65.3, |
| "ece": 0.083 |
| }, |
| "v0.6": { |
| "acc": 65.9, |
| "ece": 0.054 |
| }, |
| "delta": 0.6, |
| "ci95": [ |
| -1.3, |
| 2.4 |
| ], |
| "fixes_breaks": [ |
| 49, |
| 43 |
| ], |
| "report_only": true |
| } |
| }, |
| "pr_label_questions": { |
| "pr-labels_test:type": { |
| "n": 953, |
| "v0.5": 89.4, |
| "v0.6": 85.3, |
| "delta": -4.1, |
| "ci95": [ |
| -6.0, |
| -2.3 |
| ] |
| }, |
| "pr-labels_test:blast": { |
| "n": 139, |
| "v0.5": 57.6, |
| "v0.6": 61.9, |
| "delta": 4.3, |
| "ci95": [ |
| -2.2, |
| 10.8 |
| ] |
| }, |
| "pr-labels_test:sev": { |
| "n": 953, |
| "v0.5": 76.7, |
| "v0.6": 77.9, |
| "delta": 1.2, |
| "ci95": [ |
| -1.3, |
| 3.6 |
| ] |
| }, |
| "pr-labels_development:type": { |
| "n": 946, |
| "v0.5": 90.8, |
| "v0.6": 90.7, |
| "delta": -0.1, |
| "ci95": [ |
| -1.8, |
| 1.5 |
| ] |
| }, |
| "pr-labels_development:blast": { |
| "n": 138, |
| "v0.5": 58.7, |
| "v0.6": 52.9, |
| "delta": -5.8, |
| "ci95": [ |
| -11.6, |
| 0.0 |
| ] |
| }, |
| "pr-labels_development:sev": { |
| "n": 946, |
| "v0.5": 74.3, |
| "v0.6": 72.5, |
| "delta": -1.8, |
| "ci95": [ |
| -3.9, |
| 0.3 |
| ] |
| }, |
| "pr-labels_test-ja:type": { |
| "n": 92, |
| "v0.5": 83.7, |
| "v0.6": 79.3, |
| "delta": -4.3, |
| "ci95": [ |
| -9.8, |
| 0.0 |
| ] |
| }, |
| "pr-labels_test-ja:blast": { |
| "n": 9, |
| "v0.5": 44.4, |
| "v0.6": 44.4, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "pr-labels_test-ja:sev": { |
| "n": 92, |
| "v0.5": 71.7, |
| "v0.6": 68.5, |
| "delta": -3.3, |
| "ci95": [ |
| -10.9, |
| 4.3 |
| ] |
| }, |
| "pr-labels_development-ja:type": { |
| "n": 95, |
| "v0.5": 87.4, |
| "v0.6": 85.3, |
| "delta": -2.1, |
| "ci95": [ |
| -6.3, |
| 1.1 |
| ] |
| }, |
| "pr-labels_development-ja:blast": { |
| "n": 8, |
| "v0.5": 62.5, |
| "v0.6": 50.0, |
| "delta": -12.5, |
| "ci95": [ |
| -37.5, |
| 0.0 |
| ] |
| }, |
| "pr-labels_development-ja:sev": { |
| "n": 95, |
| "v0.5": 68.4, |
| "v0.6": 70.5, |
| "delta": 2.1, |
| "ci95": [ |
| -4.2, |
| 7.4 |
| ] |
| }, |
| "pr-labels_real17:type": { |
| "n": 17, |
| "v0.5": 82.4, |
| "v0.6": 82.4, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "pr-labels_real17:blast": { |
| "n": 17, |
| "v0.5": 58.8, |
| "v0.6": 70.6, |
| "delta": 11.8, |
| "ci95": [ |
| 0.0, |
| 29.4 |
| ] |
| }, |
| "pr-labels_real17:sev": { |
| "n": 17, |
| "v0.5": 52.9, |
| "v0.6": 52.9, |
| "delta": 0.0, |
| "ci95": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "pr-labels_test:sev+offsets": { |
| "n": 953, |
| "v0.5": 77.0, |
| "v0.6": 76.6, |
| "delta": -0.4, |
| "ci95": [ |
| -2.4, |
| 1.8 |
| ] |
| } |
| }, |
| "fresh_prs_eval": { |
| "sev": { |
| "n": 487, |
| "majority_floor": 0.678, |
| "acc": { |
| "v0.4": 0.774, |
| "v0.5": 0.747, |
| "v0.6": 0.791 |
| }, |
| "v0.6_vs_v0.5": { |
| "delta": 4.3, |
| "ci95": [ |
| 0.8, |
| 7.8 |
| ], |
| "fixes": 52, |
| "breaks": 31, |
| "sign_p": 0.028 |
| }, |
| "v0.6_vs_v0.4": { |
| "delta": 1.6, |
| "ci95": [ |
| -0.8, |
| 4.1 |
| ] |
| }, |
| "recall": { |
| "v0.4": { |
| "P0": 0.29, |
| "P1": 0.35, |
| "P2": 0.58, |
| "P3": 0.89, |
| "P4": 0.5 |
| }, |
| "v0.5": { |
| "P0": 0.29, |
| "P1": 0.22, |
| "P2": 0.79, |
| "P3": 0.78, |
| "P4": 0.5 |
| }, |
| "v0.6": { |
| "P0": 0.71, |
| "P1": 0.35, |
| "P2": 0.55, |
| "P3": 0.92, |
| "P4": 0.5 |
| } |
| } |
| }, |
| "type": { |
| "n": 487, |
| "majority_floor": 0.515, |
| "acc": { |
| "v0.4": 0.864, |
| "v0.5": 0.867, |
| "v0.6": 0.906 |
| }, |
| "v0.6_vs_v0.5": { |
| "delta": 3.9, |
| "ci95": [ |
| 1.2, |
| 6.6 |
| ], |
| "fixes": 31, |
| "breaks": 12, |
| "sign_p": 0.005 |
| }, |
| "v0.6_vs_v0.4": { |
| "delta": 4.1, |
| "ci95": [ |
| 1.4, |
| 7.0 |
| ] |
| } |
| } |
| } |
| } |
|
|
