eval/raw_e4b_nf4_control_summary.json

{
  "model": "google/gemma-4-E4B-it",
  "precision": "NF4",
  "readout": "native LM-head answer-label scoring",
  "complete": true,
  "task_count": 5,
  "source": "final benchmark output recorded 2026-09-26",
  "methodological_note": "Quantization-matched base control for the local path. End-to-end system comparison versus OpenJEV because the readout architecture differs.",
  "results": {
    "arc_easy": {
      "accuracy": 0.9558,
      "nll": 0.2218,
      "ece": 0.0334,
      "precision": "NF4",
      "readout": "native_lm_head_answer_label_scoring"
    },
    "arc_challenge": {
      "accuracy": 0.8737,
      "nll": 0.6831,
      "ece": 0.094,
      "precision": "NF4",
      "readout": "native_lm_head_answer_label_scoring"
    },
    "winogrande": {
      "accuracy": 0.6014,
      "nll": 1.2789,
      "ece": 0.3053,
      "precision": "NF4",
      "readout": "native_lm_head_answer_label_scoring"
    },
    "hellaswag": {
      "accuracy": 0.7651,
      "nll": 1.1007,
      "ece": 0.1655,
      "precision": "NF4",
      "readout": "native_lm_head_answer_label_scoring"
    },
    "mmlu": {
      "accuracy": 0.6619,
      "nll": 1.6791,
      "ece": 0.2418,
      "precision": "NF4",
      "readout": "native_lm_head_answer_label_scoring"
    }
  }
}