benchmark_results.json

{
  "type": "aggregate_r2a_regression_comparison",
  "checkpoint": "R2a",
  "models": [
    {
      "model": "laya-cybersec-r2a",
      "name": "Laya R2a",
      "full_en": {
        "n": 510,
        "attacks": 256,
        "clean": 254,
        "auroc": 0.9155004306102362,
        "caught": 140,
        "false_alarms": 2,
        "recall": 0.546875,
        "fpr": 0.007874015748031496
      },
      "full_de": {
        "n": 510,
        "attacks": 256,
        "clean": 254,
        "auroc": 0.8780065821850394,
        "caught": 151,
        "false_alarms": 9,
        "recall": 0.58984375,
        "fpr": 0.03543307086614173
      },
      "skills_en": {
        "n": 48,
        "attacks": 27,
        "clean": 21,
        "auroc": 0.927689594356261,
        "caught": 18,
        "false_alarms": 0
      },
      "skills_de": {
        "n": 48,
        "attacks": 27,
        "clean": 21,
        "auroc": 0.9329805996472663,
        "caught": 18,
        "false_alarms": 0
      },
      "pdf": {
        "n": 730,
        "attacks": 107,
        "clean": 623,
        "auroc": 0.8855927753859077,
        "caught": 81,
        "false_alarms": 0
      },
      "decision_rule": "score > 0.95"
    },
    {
      "model": "jev",
      "name": "Jev",
      "full_en": {
        "auroc": 0.9799920029527559,
        "n": 510
      },
      "full_de": {
        "auroc": 0.9564391609251969,
        "n": 510
      },
      "skills_en": {
        "n": 48,
        "attacks": 27,
        "clean": 21,
        "auroc": 0.9841269841269841,
        "caught": 25,
        "false_alarms": 1
      },
      "skills_de": {
        "n": 48,
        "attacks": 27,
        "clean": 21,
        "auroc": 0.9603174603174603,
        "caught": 25,
        "false_alarms": 2
      },
      "pdf": {
        "n": 730,
        "attacks": 107,
        "clean": 623,
        "auroc": 0.978525674682348,
        "caught": 73,
        "false_alarms": 2
      },
      "decision_rule": "score > 0.5"
    },
    {
      "model": "TextCortex/clef-cybersecurity",
      "name": "clef-cybersecurity",
      "full_en": {
        "n": 510,
        "attacks": 256,
        "clean": 254,
        "auroc": 0.9925335260826772,
        "caught": 232,
        "false_alarms": 0
      },
      "full_de": {
        "n": 510,
        "attacks": 256,
        "clean": 254,
        "auroc": 0.9743940698818898,
        "caught": 241,
        "false_alarms": 17
      },
      "skills_en": {
        "n": 48,
        "attacks": 27,
        "clean": 21,
        "auroc": 1.0,
        "caught": 23,
        "false_alarms": 0
      },
      "skills_de": {
        "n": 48,
        "attacks": 27,
        "clean": 21,
        "auroc": 0.9171075837742504,
        "caught": 26,
        "false_alarms": 3
      },
      "pdf": {
        "n": 730,
        "attacks": 107,
        "clean": 623,
        "auroc": 0.9856212778086137,
        "caught": 84,
        "false_alarms": 3
      },
      "decision_rule": "score > 0.5"
    },
    {
      "model": "Cloudflare/clef-flash",
      "name": "CLEF Flash (base)",
      "full_en": {
        "n": 510,
        "attacks": 256,
        "clean": 254,
        "auroc": 0.9587613804133859,
        "caught": 136,
        "false_alarms": 0
      },
      "full_de": {
        "n": 510,
        "attacks": 256,
        "clean": 254,
        "auroc": 0.9391455462598425,
        "caught": 130,
        "false_alarms": 0
      },
      "skills_en": {
        "n": 48,
        "attacks": 27,
        "clean": 21,
        "auroc": 0.9761904761904762,
        "caught": 14,
        "false_alarms": 0
      },
      "skills_de": {
        "n": 48,
        "attacks": 27,
        "clean": 21,
        "auroc": 0.9047619047619048,
        "caught": 12,
        "false_alarms": 0
      },
      "pdf": {
        "n": 730,
        "attacks": 107,
        "clean": 623,
        "auroc": 0.8144492281843957,
        "caught": 6,
        "false_alarms": 0
      },
      "decision_rule": "score > 0.5"
    }
  ],
  "cohort": {
    "full_per_language": 510,
    "skills_per_language": 48,
    "pdf_attacks": 107,
    "clean_pdfs": 623
  },
  "scope": "Previously inspected internal regression sets; not a public leaderboard or fresh blind test.",
  "threshold_note": "Strict >0.5 for both CLEF models and Jev; strict >0.95 for Laya R2a. Recall and false alarms are not equal-threshold comparisons.",
  "backend_note": "R2a full-suite results use the saved batched GPU run; matched skill/PDF references use saved local scores. Backend precision causes small score differences; GPU English-skills AUROC was 0.9268077601410935.",
  "base_clef_evaluation": {
    "type": "unchanged_local_base",
    "model": "Cloudflare/clef-flash",
    "revision": "17f0b0ad64efb65d273590632833508766b2aae6",
    "scope": "Unchanged native checkpoint, evaluated locally on the matched cohorts with token-bounded windows. Not the Cloudflare hosted API run."
  }
}