File size: 5,720 Bytes
e55fce6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | {
"schema_version": 1,
"report": "e3-pilot",
"generated_at_utc": "2026-09-12",
"suite_id": "e3-humanevalplus-pilot-v1",
"rows": {
"original-bf16": {
"generation_evidence": "reports/e3/e3-pilot-20260912T131807Z",
"generation_summary": {
"task_count": 33,
"generation_record_count": 33,
"terminal_record_count": 33,
"retried_task_count": 0,
"infrastructure_failure_count": 0,
"duplicate_generation_keys": 0,
"total_completion_tokens": 52186,
"generated_programs_executed": 0
},
"extraction_yield": 0.848485,
"failed_extraction_task_ids": [
"HumanEval/148",
"HumanEval/163",
"HumanEval/32",
"HumanEval/52",
"HumanEval/57"
],
"scoring_evidence": "reports/e3/e3-pilot-20260912T131807Z/original-bf16/scoring/e3-score-20260912T133052406Z",
"pass_at_1_base": 0.75,
"pass_at_1_plus": 0.7142857142857143,
"scores_computed": 28,
"mean_decode_tokens_per_second": 112.3
},
"merged-bf16": {
"generation_evidence": "reports/e3/e3-pilot-20260912T131705Z",
"generation_summary": {
"task_count": 33,
"generation_record_count": 33,
"terminal_record_count": 33,
"retried_task_count": 0,
"infrastructure_failure_count": 0,
"duplicate_generation_keys": 0,
"total_completion_tokens": 3578,
"generated_programs_executed": 0
},
"extraction_yield": 1.0,
"failed_extraction_task_ids": [],
"scoring_evidence": "reports/e3/e3-pilot-20260912T131705Z/merged-bf16/scoring/e3-score-20260912T133017638Z",
"pass_at_1_base": 0.36363636363636365,
"pass_at_1_plus": 0.2727272727272727,
"scores_computed": 33,
"mean_decode_tokens_per_second": 122.1
},
"q8_0": {
"generation_evidence": "reports/e3/e3-pilot-20260912T132635Z",
"generation_summary": {
"task_count": 33,
"generation_record_count": 33,
"terminal_record_count": 33,
"retried_task_count": 0,
"infrastructure_failure_count": 0,
"duplicate_generation_keys": 0,
"total_completion_tokens": 3582,
"generated_programs_executed": 0
},
"extraction_yield": 1.0,
"failed_extraction_task_ids": [],
"scoring_evidence": "reports/e3/e3-pilot-20260912T132635Z/q8_0/scoring/e3-score-20260912T133124723Z",
"pass_at_1_base": 0.36363636363636365,
"pass_at_1_plus": 0.2727272727272727,
"scores_computed": 33,
"mean_decode_tokens_per_second": 197.2
},
"q6_k": {
"generation_evidence": "reports/e3/e3-pilot-20260912T132714Z",
"generation_summary": {
"task_count": 33,
"generation_record_count": 33,
"terminal_record_count": 33,
"retried_task_count": 0,
"infrastructure_failure_count": 0,
"duplicate_generation_keys": 0,
"total_completion_tokens": 3541,
"generated_programs_executed": 0
},
"extraction_yield": 1.0,
"failed_extraction_task_ids": [],
"scoring_evidence": "reports/e3/e3-pilot-20260912T132714Z/q6_k/scoring/e3-score-20260912T133148491Z",
"pass_at_1_base": 0.36363636363636365,
"pass_at_1_plus": 0.30303030303030304,
"scores_computed": 33,
"mean_decode_tokens_per_second": 201.8
}
},
"paired_deltas": {
"fine_tune_effect_plus_original_to_merged": {
"gained": [],
"lost": [
"HumanEval/1",
"HumanEval/102",
"HumanEval/107",
"HumanEval/128",
"HumanEval/133",
"HumanEval/138",
"HumanEval/158",
"HumanEval/37",
"HumanEval/62",
"HumanEval/72",
"HumanEval/87",
"HumanEval/92"
]
},
"quant_effect_plus_merged_to_q8_0": {
"lost": []
},
"quant_effect_plus_merged_to_q6_k": {
"lost": []
}
},
"findings": {
"harness_health": {
"infrastructure_failures": 0,
"retried_tasks": 0,
"duplicate_keys": 0,
"rows_completed": 4,
"generated_programs_executed_host": 0
},
"fine_tune_effect": "Negative on this pilot slice: original plus 0.714 (28 scored, yield 0.848 -> at most 0.606 if all unextractable scored zero) vs merged plus 0.273 (33 scored). Twelve paired losses, zero gains.",
"quantization_effect": "Essentially lossless at pilot scale: Q8_0 identical to merged; Q6_K no paired losses and one plus gain.",
"caveats": [
"n=33 tasks, one greedy sample per task; exact deltas carry slice noise until the sealed 128-task run.",
"Original-row extraction yield 0.848 reflects reasoning-leakage outputs of the base thinking model under prompt v1; absence scores zero, which slightly favors the fine-tune in the paired delta and cannot explain it.",
"KodCode contamination caveat applies to absolute scores; the paired delta remains the scientific comparison."
]
},
"frozen_e4_decisions": {
"sealed_model_rows": [
"original-bf16",
"merged-bf16",
"q8_0",
"q6_k"
],
"deployment_quant": "q6_k (no paired loss vs merged BF16, one plus gain, 963 MB vs 1246 MB)",
"deployment_quant_gate": "Q6_K deployable iff sealed pass@1(plus) within 2 pp of merged-bf16 AND no paired loss pattern absent from merged; frozen before any sealed request.",
"harness_health_gates": "Infra failure rate <= 2% of requests; extraction yield reported per row as model evidence, not a harness pass gate (original-row 0.848 is a model property)."
}
}
|