File size: 7,751 Bytes
b440dc9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | {
"schema_version": 1,
"report": "e4-sealed",
"generated_at_utc": "2026-09-12",
"suite_id": "e4-humanevalplus-sealed-v1",
"rows": {
"original-bf16": {
"session": "e4-sealed-20260912T135909Z",
"generation_summary": {
"task_count": 128,
"generation_record_count": 128,
"terminal_record_count": 128,
"retried_task_count": 0,
"infrastructure_failure_count": 0,
"duplicate_generation_keys": 0,
"total_completion_tokens": 211732,
"generated_programs_executed": 0
},
"extraction_yield": 0.789062,
"failed_extraction_count": 27,
"scoring_evidence": "reports/e4/e4-sealed-20260912T135909Z/original-bf16/scoring/e3-score-20260912T143512162Z",
"scored_task_count": 101,
"pass_at_1_base": 0.6138613861386139,
"pass_at_1_plus": 0.5544554455445545,
"mean_decode_tokens_per_second": 101.3,
"pass_count_plus": 56,
"pass_count_base": 62,
"plus_rate_absence_as_zero": 0.4375,
"base_rate_absence_as_zero": 0.4844
},
"merged-bf16": {
"session": "e4-sealed-20260912T135057Z",
"generation_summary": {
"task_count": 128,
"generation_record_count": 128,
"terminal_record_count": 128,
"retried_task_count": 0,
"infrastructure_failure_count": 0,
"duplicate_generation_keys": 0,
"total_completion_tokens": 18992,
"generated_programs_executed": 0
},
"extraction_yield": 0.984375,
"failed_extraction_count": 2,
"scoring_evidence": "reports/e4/e4-sealed-20260912T135057Z/merged-bf16/scoring/e3-score-20260912T135917459Z",
"scored_task_count": 126,
"pass_at_1_base": 0.46825396825396826,
"pass_at_1_plus": 0.40476190476190477,
"mean_decode_tokens_per_second": 113.6,
"pass_count_plus": 51,
"pass_count_base": 59,
"plus_rate_absence_as_zero": 0.3984,
"base_rate_absence_as_zero": 0.4609
},
"q8_0": {
"session": "e4-sealed-20260912T135437Z",
"generation_summary": {
"task_count": 128,
"generation_record_count": 128,
"terminal_record_count": 128,
"retried_task_count": 0,
"infrastructure_failure_count": 0,
"duplicate_generation_keys": 0,
"total_completion_tokens": 16425,
"generated_programs_executed": 0
},
"extraction_yield": 0.992188,
"failed_extraction_count": 1,
"scoring_evidence": "reports/e4/e4-sealed-20260912T135437Z/q8_0/scoring/e3-score-20260912T140124851Z",
"scored_task_count": 127,
"pass_at_1_base": 0.4645669291338583,
"pass_at_1_plus": 0.41732283464566927,
"mean_decode_tokens_per_second": 181.8,
"pass_count_plus": 53,
"pass_count_base": 59,
"plus_rate_absence_as_zero": 0.4141,
"base_rate_absence_as_zero": 0.4609
},
"q6_k": {
"session": "e4-sealed-20260912T135648Z",
"generation_summary": {
"task_count": 128,
"generation_record_count": 128,
"terminal_record_count": 128,
"retried_task_count": 0,
"infrastructure_failure_count": 0,
"duplicate_generation_keys": 0,
"total_completion_tokens": 19346,
"generated_programs_executed": 0
},
"extraction_yield": 0.984375,
"failed_extraction_count": 2,
"scoring_evidence": "reports/e4/e4-sealed-20260912T135648Z/q6_k/scoring/e3-score-20260912T140241797Z",
"scored_task_count": 126,
"pass_at_1_base": 0.42857142857142855,
"pass_at_1_plus": 0.3888888888888889,
"mean_decode_tokens_per_second": 188.5,
"pass_count_plus": 49,
"pass_count_base": 54,
"plus_rate_absence_as_zero": 0.3828,
"base_rate_absence_as_zero": 0.4219
}
},
"paired_deltas": {
"note": "Paired comparisons use only tasks scored in both rows (extraction failures score zero by absence).",
"original_vs_merged": {
"common_scored_tasks": 99,
"merged_plus_pass_count": 49,
"original_plus_pass_count": 56,
"fine_tune_gained": [
"HumanEval/104",
"HumanEval/105",
"HumanEval/12",
"HumanEval/13",
"HumanEval/149",
"HumanEval/21",
"HumanEval/23",
"HumanEval/25",
"HumanEval/28",
"HumanEval/35",
"HumanEval/5",
"HumanEval/53",
"HumanEval/59",
"HumanEval/60",
"HumanEval/61",
"HumanEval/8"
],
"fine_tune_lost": [
"HumanEval/100",
"HumanEval/103",
"HumanEval/110",
"HumanEval/111",
"HumanEval/121",
"HumanEval/127",
"HumanEval/139",
"HumanEval/141",
"HumanEval/151",
"HumanEval/152",
"HumanEval/156",
"HumanEval/159",
"HumanEval/161",
"HumanEval/24",
"HumanEval/36",
"HumanEval/41",
"HumanEval/49",
"HumanEval/69",
"HumanEval/70",
"HumanEval/71",
"HumanEval/81",
"HumanEval/95",
"HumanEval/98"
]
},
"merged_vs_q8_0": {
"common_scored_tasks": 126,
"quant_lost_vs_merged": [
"HumanEval/105"
],
"quant_gained_vs_merged": [
"HumanEval/88",
"HumanEval/89"
]
},
"merged_vs_q6_k": {
"common_scored_tasks": 126,
"quant_lost_vs_merged": [
"HumanEval/105",
"HumanEval/146",
"HumanEval/80"
],
"quant_gained_vs_merged": [
"HumanEval/70"
]
}
},
"deployment_gate_verdict": {
"selected_quant": "q6_k",
"gate_frozen": "Q6_K deployable iff sealed pass@1(plus) within 2 pp of merged-bf16 AND no new paired failure class absent from merged-bf16",
"delta_vs_merged_pp": -1.59,
"paired_losses": [
"HumanEval/105",
"HumanEval/146",
"HumanEval/80"
],
"paired_gains": [
"HumanEval/70"
],
"assessment": "PASS: -1.59 pp (within the 2 pp gate); three scattered single-task losses with one gain is quantization noise near token ties, not a new failure class. Q8_0 is quality-equivalent-or-better (net +1 task) but 296 MB larger; Q6_K remains the frozen selection.",
"runner_up": {
"alias": "q8_0",
"note": "net +1 paired task vs merged-bf16 at 1246 MB"
}
},
"findings": {
"fine_tune_effect": "Negative, but smaller than the pilot suggested: on the 99 tasks scored in both rows the fine-tune passes 49 versus the original 56 (16 gained, 23 lost, net -7, -7.1 pp paired). Counting every unextractable output as a deployed failure: original 0.438 vs merged 0.398 of 128.",
"quantization_effect": "Q8_0 and Q6_K stay within noise of merged BF16 on paired tasks.",
"original_reasoning_leakage": "The original model emitted 211,732 completion tokens (about 1,654 per task) with 27 outputs unextractable; the fine-tune emitted 18,992 tokens (about 148 per task) with 2 unextractable.",
"harness_health": "Zero infrastructure failures, zero retries, zero duplicate keys across all four rows."
},
"caveats": [
"Scorer pass@1 excludes unextractable outputs from the denominator; plus_rate_absence_as_zero is the deployment-honest rate and is reported alongside.",
"One greedy sample per task; no seed variance measured (deterministic regression track).",
"KodCode contamination caveat applies to absolute scores; paired deltas remain the scientific comparison.",
"HumanEval/105 is a shared quantization-sensitive task lost by both quants."
]
}
|