{
  "schema_version": 1,
  "report": "e4-sealed",
  "generated_at_utc": "2026-09-12",
  "suite_id": "e4-humanevalplus-sealed-v1",
  "rows": {
    "original-bf16": {
      "session": "e4-sealed-20260912T135909Z",
      "generation_summary": {
        "task_count": 128,
        "generation_record_count": 128,
        "terminal_record_count": 128,
        "retried_task_count": 0,
        "infrastructure_failure_count": 0,
        "duplicate_generation_keys": 0,
        "total_completion_tokens": 211732,
        "generated_programs_executed": 0
      },
      "extraction_yield": 0.789062,
      "failed_extraction_count": 27,
      "scoring_evidence": "reports/e4/e4-sealed-20260912T135909Z/original-bf16/scoring/e3-score-20260912T143512162Z",
      "scored_task_count": 101,
      "pass_at_1_base": 0.6138613861386139,
      "pass_at_1_plus": 0.5544554455445545,
      "mean_decode_tokens_per_second": 101.3,
      "pass_count_plus": 56,
      "pass_count_base": 62,
      "plus_rate_absence_as_zero": 0.4375,
      "base_rate_absence_as_zero": 0.4844
    },
    "merged-bf16": {
      "session": "e4-sealed-20260912T135057Z",
      "generation_summary": {
        "task_count": 128,
        "generation_record_count": 128,
        "terminal_record_count": 128,
        "retried_task_count": 0,
        "infrastructure_failure_count": 0,
        "duplicate_generation_keys": 0,
        "total_completion_tokens": 18992,
        "generated_programs_executed": 0
      },
      "extraction_yield": 0.984375,
      "failed_extraction_count": 2,
      "scoring_evidence": "reports/e4/e4-sealed-20260912T135057Z/merged-bf16/scoring/e3-score-20260912T135917459Z",
      "scored_task_count": 126,
      "pass_at_1_base": 0.46825396825396826,
      "pass_at_1_plus": 0.40476190476190477,
      "mean_decode_tokens_per_second": 113.6,
      "pass_count_plus": 51,
      "pass_count_base": 59,
      "plus_rate_absence_as_zero": 0.3984,
      "base_rate_absence_as_zero": 0.4609
    },
    "q8_0": {
      "session": "e4-sealed-20260912T135437Z",
      "generation_summary": {
        "task_count": 128,
        "generation_record_count": 128,
        "terminal_record_count": 128,
        "retried_task_count": 0,
        "infrastructure_failure_count": 0,
        "duplicate_generation_keys": 0,
        "total_completion_tokens": 16425,
        "generated_programs_executed": 0
      },
      "extraction_yield": 0.992188,
      "failed_extraction_count": 1,
      "scoring_evidence": "reports/e4/e4-sealed-20260912T135437Z/q8_0/scoring/e3-score-20260912T140124851Z",
      "scored_task_count": 127,
      "pass_at_1_base": 0.4645669291338583,
      "pass_at_1_plus": 0.41732283464566927,
      "mean_decode_tokens_per_second": 181.8,
      "pass_count_plus": 53,
      "pass_count_base": 59,
      "plus_rate_absence_as_zero": 0.4141,
      "base_rate_absence_as_zero": 0.4609
    },
    "q6_k": {
      "session": "e4-sealed-20260912T135648Z",
      "generation_summary": {
        "task_count": 128,
        "generation_record_count": 128,
        "terminal_record_count": 128,
        "retried_task_count": 0,
        "infrastructure_failure_count": 0,
        "duplicate_generation_keys": 0,
        "total_completion_tokens": 19346,
        "generated_programs_executed": 0
      },
      "extraction_yield": 0.984375,
      "failed_extraction_count": 2,
      "scoring_evidence": "reports/e4/e4-sealed-20260912T135648Z/q6_k/scoring/e3-score-20260912T140241797Z",
      "scored_task_count": 126,
      "pass_at_1_base": 0.42857142857142855,
      "pass_at_1_plus": 0.3888888888888889,
      "mean_decode_tokens_per_second": 188.5,
      "pass_count_plus": 49,
      "pass_count_base": 54,
      "plus_rate_absence_as_zero": 0.3828,
      "base_rate_absence_as_zero": 0.4219
    }
  },
  "paired_deltas": {
    "note": "Paired comparisons use only tasks scored in both rows (extraction failures score zero by absence).",
    "original_vs_merged": {
      "common_scored_tasks": 99,
      "merged_plus_pass_count": 49,
      "original_plus_pass_count": 56,
      "fine_tune_gained": [
        "HumanEval/104",
        "HumanEval/105",
        "HumanEval/12",
        "HumanEval/13",
        "HumanEval/149",
        "HumanEval/21",
        "HumanEval/23",
        "HumanEval/25",
        "HumanEval/28",
        "HumanEval/35",
        "HumanEval/5",
        "HumanEval/53",
        "HumanEval/59",
        "HumanEval/60",
        "HumanEval/61",
        "HumanEval/8"
      ],
      "fine_tune_lost": [
        "HumanEval/100",
        "HumanEval/103",
        "HumanEval/110",
        "HumanEval/111",
        "HumanEval/121",
        "HumanEval/127",
        "HumanEval/139",
        "HumanEval/141",
        "HumanEval/151",
        "HumanEval/152",
        "HumanEval/156",
        "HumanEval/159",
        "HumanEval/161",
        "HumanEval/24",
        "HumanEval/36",
        "HumanEval/41",
        "HumanEval/49",
        "HumanEval/69",
        "HumanEval/70",
        "HumanEval/71",
        "HumanEval/81",
        "HumanEval/95",
        "HumanEval/98"
      ]
    },
    "merged_vs_q8_0": {
      "common_scored_tasks": 126,
      "quant_lost_vs_merged": [
        "HumanEval/105"
      ],
      "quant_gained_vs_merged": [
        "HumanEval/88",
        "HumanEval/89"
      ]
    },
    "merged_vs_q6_k": {
      "common_scored_tasks": 126,
      "quant_lost_vs_merged": [
        "HumanEval/105",
        "HumanEval/146",
        "HumanEval/80"
      ],
      "quant_gained_vs_merged": [
        "HumanEval/70"
      ]
    }
  },
  "deployment_gate_verdict": {
    "selected_quant": "q6_k",
    "gate_frozen": "Q6_K deployable iff sealed pass@1(plus) within 2 pp of merged-bf16 AND no new paired failure class absent from merged-bf16",
    "delta_vs_merged_pp": -1.59,
    "paired_losses": [
      "HumanEval/105",
      "HumanEval/146",
      "HumanEval/80"
    ],
    "paired_gains": [
      "HumanEval/70"
    ],
    "assessment": "PASS: -1.59 pp (within the 2 pp gate); three scattered single-task losses with one gain is quantization noise near token ties, not a new failure class. Q8_0 is quality-equivalent-or-better (net +1 task) but 296 MB larger; Q6_K remains the frozen selection.",
    "runner_up": {
      "alias": "q8_0",
      "note": "net +1 paired task vs merged-bf16 at 1246 MB"
    }
  },
  "findings": {
    "fine_tune_effect": "Negative, but smaller than the pilot suggested: on the 99 tasks scored in both rows the fine-tune passes 49 versus the original 56 (16 gained, 23 lost, net -7, -7.1 pp paired). Counting every unextractable output as a deployed failure: original 0.438 vs merged 0.398 of 128.",
    "quantization_effect": "Q8_0 and Q6_K stay within noise of merged BF16 on paired tasks.",
    "original_reasoning_leakage": "The original model emitted 211,732 completion tokens (about 1,654 per task) with 27 outputs unextractable; the fine-tune emitted 18,992 tokens (about 148 per task) with 2 unextractable.",
    "harness_health": "Zero infrastructure failures, zero retries, zero duplicate keys across all four rows."
  },
  "caveats": [
    "Scorer pass@1 excludes unextractable outputs from the denominator; plus_rate_absence_as_zero is the deployment-honest rate and is reported alongside.",
    "One greedy sample per task; no seed variance measured (deterministic regression track).",
    "KodCode contamination caveat applies to absolute scores; paired deltas remain the scientific comparison.",
    "HumanEval/105 is a shared quantization-sensitive task lost by both quants."
  ]
}