{
  "date": "2026-09-30",
  "metric": "argmax_accuracy",
  "models": {
    "jev": {
      "description": "Hosted service; JevBench tested locally at Jev 1.13.0, other suites from Kev authors"
    },
    "kev_9b": {
      "revision": "2629c06a",
      "code_revision": "557598fced1dada75dfbf36ed144dce309ac6ceb",
      "dtype": "fp32",
      "temperature": 1.0,
      "source": "JevBench locally rerun; other suites from author reports"
    },
    "qev_9b": {
      "repo_id": "AustinFu/Qev-9B",
      "revision": "v0.2.0",
      "base": "Qwen/Qwen3.5-9B-Base",
      "base_revision": "68c46c4b3498877f3ef123c856ecfde50c39f404",
      "step": 2643,
      "dtype": "bf16",
      "execution": "reference",
      "seed": 17,
      "temperature": 1.0
    },
    "qwen35_9b_base": {
      "description": "Frozen native LM head; zero-shot candidate code probabilities",
      "dtype": "bf16"
    },
    "qev_2b": {
      "base": "Qwen/Qwen3.5-2B-Base",
      "base_revision": "b1485b2fa6dfa1287294f269f5fb618e03d52d7c",
      "dtype": "bf16",
      "description": "Probability and representation-response distillation from Qev-9B v0.1.0",
      "seed": 17,
      "continuation_steps": 800,
      "teacher_revision": "v0.1.0"
    },
    "qwen35_2b_base": {
      "description": "Frozen native LM head; zero-shot candidate code probabilities",
      "dtype": "bf16",
      "base": "Qwen/Qwen3.5-2B-Base",
      "base_revision": "b1485b2fa6dfa1287294f269f5fb618e03d52d7c"
    }
  },
  "results": {
    "decision_dev_all": {
      "total": 1468,
      "models": {
        "jev": {
          "correct": 1221,
          "accuracy": 0.8317438692098093
        },
        "kev_9b": {
          "correct": 1289,
          "accuracy": 0.8780653950953679
        },
        "qev_9b": {
          "correct": 1295,
          "accuracy": 0.8821525885558583
        },
        "qwen35_9b_base": {
          "correct": 1112,
          "accuracy": 0.7574931880108992
        },
        "qev_2b": {
          "correct": 1266,
          "accuracy": 0.8623978201634878
        },
        "qwen35_2b_base": {
          "correct": 927,
          "accuracy": 0.6314713896457765
        }
      }
    },
    "decision_dev_clean": {
      "total": 1264,
      "models": {
        "jev": {
          "correct": 1068,
          "accuracy": 0.8449367088607594
        },
        "kev_9b": {
          "correct": 1102,
          "accuracy": 0.8718354430379747
        },
        "qev_9b": {
          "correct": 1105,
          "accuracy": 0.8742088607594937
        },
        "qwen35_9b_base": {
          "correct": 982,
          "accuracy": 0.7768987341772152
        },
        "qev_2b": {
          "correct": 1079,
          "accuracy": 0.8536392405063291
        },
        "qwen35_2b_base": {
          "correct": 827,
          "accuracy": 0.6542721518987342
        }
      }
    },
    "transfer_dev_all": {
      "total": 764,
      "models": {
        "jev": {
          "correct": 647,
          "accuracy": 0.8468586387434555
        },
        "kev_9b": {
          "correct": 620,
          "accuracy": 0.8115183246073299
        },
        "qev_9b": {
          "correct": 630,
          "accuracy": 0.824607329842932
        },
        "qwen35_9b_base": {
          "correct": 561,
          "accuracy": 0.7342931937172775
        },
        "qev_2b": {
          "correct": 584,
          "accuracy": 0.7643979057591623
        },
        "qwen35_2b_base": {
          "correct": 493,
          "accuracy": 0.6452879581151832
        }
      }
    },
    "transfer_dev_clean": {
      "total": 656,
      "models": {
        "jev": {
          "correct": 562,
          "accuracy": 0.8567073170731707
        },
        "kev_9b": {
          "correct": 539,
          "accuracy": 0.8216463414634146
        },
        "qev_9b": {
          "correct": 551,
          "accuracy": 0.8399390243902439
        },
        "qwen35_9b_base": {
          "correct": 488,
          "accuracy": 0.7439024390243902
        },
        "qev_2b": {
          "correct": 507,
          "accuracy": 0.7728658536585366
        },
        "qwen35_2b_base": {
          "correct": 427,
          "accuracy": 0.6509146341463414
        }
      }
    },
    "mmlupro": {
      "total": 1000,
      "models": {
        "jev": {
          "correct": 835,
          "accuracy": 0.835
        },
        "kev_9b": {
          "correct": 511,
          "accuracy": 0.511
        },
        "qev_9b": {
          "correct": 574,
          "accuracy": 0.574
        },
        "qwen35_9b_base": {
          "correct": 504,
          "accuracy": 0.504
        },
        "qev_2b": {
          "correct": 387,
          "accuracy": 0.387
        },
        "qwen35_2b_base": {
          "correct": 312,
          "accuracy": 0.312
        }
      }
    },
    "semif_handwritten": {
      "total": 144,
      "models": {
        "jev": {
          "correct": 139,
          "accuracy": 0.9652777777777778
        },
        "kev_9b": {
          "correct": 131,
          "accuracy": 0.9097222222222222
        },
        "qev_9b": {
          "correct": 134,
          "accuracy": 0.9305555555555556
        },
        "qwen35_9b_base": {
          "correct": 130,
          "accuracy": 0.9027777777777778
        },
        "qev_2b": {
          "correct": 119,
          "accuracy": 0.8263888888888888
        },
        "qwen35_2b_base": {
          "correct": 92,
          "accuracy": 0.6388888888888888
        }
      }
    },
    "scienthoon": {
      "total": 873,
      "models": {
        "jev": {
          "correct": 657,
          "accuracy": 0.7525773195876289
        },
        "kev_9b": {
          "correct": 659,
          "accuracy": 0.7548682703321878
        },
        "qev_9b": {
          "correct": 620,
          "accuracy": 0.7101947308132875
        },
        "qwen35_9b_base": {
          "correct": 601,
          "accuracy": 0.6884306987399771
        },
        "qev_2b": {
          "correct": 628,
          "accuracy": 0.7193585337915235
        },
        "qwen35_2b_base": {
          "correct": 470,
          "accuracy": 0.5383734249713631
        }
      }
    },
    "wanli": {
      "total": 256,
      "models": {
        "jev": {
          "correct": 194,
          "accuracy": 0.7578125
        },
        "kev_9b": {
          "correct": 180,
          "accuracy": 0.703125
        },
        "qev_9b": {
          "correct": 182,
          "accuracy": 0.7109375
        },
        "qwen35_9b_base": {
          "correct": 174,
          "accuracy": 0.6796875
        },
        "qev_2b": {
          "correct": 173,
          "accuracy": 0.67578125
        },
        "qwen35_2b_base": {
          "correct": 129,
          "accuracy": 0.50390625
        }
      }
    },
    "jevbench_public": {
      "total": 231,
      "models": {
        "jev": {
          "correct": 198,
          "accuracy": 0.8571428571428571
        },
        "kev_9b": {
          "correct": 175,
          "accuracy": 0.7575757575757576
        },
        "qev_9b": {
          "correct": 192,
          "accuracy": 0.8311688311688312
        },
        "qwen35_9b_base": {
          "correct": 175,
          "accuracy": 0.7575757575757576
        },
        "qev_2b": {
          "correct": 172,
          "accuracy": 0.7445887445887446
        },
        "qwen35_2b_base": {
          "correct": 146,
          "accuracy": 0.6320346320346321
        }
      }
    }
  },
  "jevbench_subsets": {
    "original": {
      "total": 72,
      "correct": {
        "jev": 71,
        "kev_9b": 65,
        "qev_9b": 68,
        "qwen35_9b_base": 59,
        "qev_2b": 64,
        "qwen35_2b_base": 44
      }
    },
    "easy": {
      "total": 48,
      "correct": {
        "jev": 48,
        "kev_9b": 48,
        "qev_9b": 48,
        "qwen35_9b_base": 48,
        "qev_2b": 48,
        "qwen35_2b_base": 48
      }
    },
    "hard": {
      "total": 111,
      "correct": {
        "jev": 79,
        "kev_9b": 62,
        "qev_9b": 76,
        "qwen35_9b_base": 68,
        "qev_2b": 60,
        "qwen35_2b_base": 54
      }
    }
  },
  "notes": [
    "Public JevBench accuracy is not the official composite score.",
    "MMLU-Pro: Kev answered 992/1000; its 8 unanswered items count as wrong.",
    "SemIf: the 144 handwritten questions, excluding the 108 perturbations.",
    "Cross-model precisions and execution implementations differ.",
    "The Qev release is one seed, and these comparisons do not isolate architecture gains.",
    "The 2B columns use the same clean development and 144-question SemIf subsets as the 9B comparison."
  ]
}