{
  "model": "riposta/CrossbowReviewer-9B",
  "metric_notes": {
    "index": "mean accuracy over the 12 rule categories (+ aggregate questions), x100",
    "bal_acc": "mean over questions of recall averaged over gold options (0.5 = chance)",
    "ece": "expected calibration error, 10 bins",
    "brier": "multi-class Brier score"
  },
  "tests": {
    "synthetic": {
      "samples": 360,
      "labels": 3271,
      "results": {
        "CrossbowReviewer-9B": {
          "index": 85.7,
          "index_ci95": [
            84.1,
            87.3
          ],
          "accuracy": 0.859,
          "bal_acc": 0.824,
          "ece": 0.016,
          "brier": 0.191
        },
        "Jev (jev-1.13.0)": {
          "index": 82.3,
          "index_ci95": [
            80.7,
            83.8
          ],
          "accuracy": 0.789,
          "bal_acc": 0.809,
          "ece": 0.043,
          "brier": 0.306
        },
        "Qwen3.5-9B-Base (no fine-tuning)": {
          "index": 70.3,
          "index_ci95": [
            68.2,
            72.5
          ],
          "accuracy": 0.69,
          "bal_acc": 0.644,
          "ece": 0.04,
          "brier": 0.424
        },
        "Claude Opus 5.5": {
          "index": 92.9,
          "index_ci95": [
            91.7,
            93.8
          ],
          "accuracy": 0.915,
          "bal_acc": 0.925,
          "ece": 0.113,
          "brier": 0.16
        },
        "GPT-6 Sol": {
          "index": 89.1,
          "index_ci95": [
            87.6,
            90.6
          ],
          "accuracy": 0.885,
          "bal_acc": 0.911,
          "ece": 0.046,
          "brier": 0.183
        }
      }
    },
    "synthetic_relabeled": {
      "samples": 359,
      "labels": 2815,
      "results": {
        "CrossbowReviewer-9B": {
          "index": 87.5,
          "index_ci95": [
            85.8,
            89.0
          ],
          "accuracy": 0.898,
          "bal_acc": 0.841,
          "ece": 0.014,
          "brier": 0.146
        },
        "Jev (jev-1.13.0)": {
          "index": 85.0,
          "index_ci95": [
            83.4,
            86.4
          ],
          "accuracy": 0.82,
          "bal_acc": 0.813,
          "ece": 0.052,
          "brier": 0.273
        },
        "Qwen3.5-9B-Base (no fine-tuning)": {
          "index": 71.4,
          "index_ci95": [
            69.2,
            73.6
          ],
          "accuracy": 0.716,
          "bal_acc": 0.648,
          "ece": 0.046,
          "brier": 0.4
        }
      }
    },
    "owasp_benchmark_v1.2": {
      "samples": 435,
      "labels": 435,
      "results": {
        "CrossbowReviewer-9B": {
          "index": 59.8,
          "index_ci95": [
            55.2,
            64.4
          ],
          "accuracy": 0.598,
          "bal_acc": 0.654,
          "ece": 0.239,
          "brier": 0.583,
          "vulnerable_flagged": 0.944,
          "safe_recognized": 0.259
        },
        "Jev (jev-1.13.0)": {
          "index": 59.8,
          "index_ci95": [
            55.2,
            64.6
          ],
          "accuracy": 0.598,
          "bal_acc": 0.653,
          "ece": 0.209,
          "brier": 0.552,
          "vulnerable_flagged": 0.967,
          "safe_recognized": 0.236
        },
        "Qwen3.5-9B-Base (no fine-tuning)": {
          "index": 64.8,
          "index_ci95": [
            60.2,
            69.0
          ],
          "accuracy": 0.648,
          "bal_acc": 0.696,
          "ece": 0.025,
          "brier": 0.441,
          "vulnerable_flagged": 0.935,
          "safe_recognized": 0.368
        },
        "Claude Opus 5.5": {
          "index": 96.8,
          "index_ci95": [
            95.2,
            98.4
          ],
          "accuracy": 0.968,
          "bal_acc": 0.979,
          "ece": 0.059,
          "brier": 0.084,
          "vulnerable_flagged": 1.0,
          "safe_recognized": 0.936
        },
        "GPT-6 Sol": {
          "index": 87.4,
          "index_ci95": [
            83.9,
            90.1
          ],
          "accuracy": 0.874,
          "bal_acc": 0.9,
          "ece": 0.106,
          "brier": 0.237,
          "vulnerable_flagged": 1.0,
          "safe_recognized": 0.75
        }
      }
    },
    "real_open_source_code": {
      "samples": 55,
      "labels": 5401,
      "results": {
        "CrossbowReviewer-9B": {
          "index": 89.5,
          "index_ci95": [
            87.9,
            91.3
          ],
          "accuracy": 0.942,
          "bal_acc": 0.783,
          "ece": 0.006,
          "brier": 0.087
        },
        "Jev (jev-1.13.0)": {
          "index": 73.2,
          "index_ci95": [
            71.0,
            75.1
          ],
          "accuracy": 0.756,
          "bal_acc": 0.717,
          "ece": 0.03,
          "brier": 0.332
        },
        "Qwen3.5-9B-Base (no fine-tuning)": {
          "index": 69.9,
          "index_ci95": [
            67.3,
            72.8
          ],
          "accuracy": 0.694,
          "bal_acc": 0.517,
          "ece": 0.031,
          "brier": 0.416
        },
        "Claude Opus 5.5": {
          "index": 92.7,
          "index_ci95": [
            90.9,
            94.5
          ],
          "accuracy": 0.962,
          "bal_acc": 0.831,
          "ece": 0.086,
          "brier": 0.073
        },
        "GPT-6 Sol": {
          "index": 95.9,
          "index_ci95": [
            94.3,
            97.2
          ],
          "accuracy": 0.979,
          "bal_acc": 0.926,
          "ece": 0.036,
          "brier": 0.043
        }
      }
    }
  }
}