{
 "model": "Wolf Defender Jev",
 "description": "33 held-out benchmarks, primary decision each; the 10 Japanese indirect-injection probe sets count as one. 'external' = source not seen during training.",
 "macro_all_33": {
  "balanced_acc": 0.9276,
  "auroc": 0.9556,
  "tpr@1%fpr": 0.7477,
  "fpr@0.5": 0.0534,
  "ece": 0.0596
 },
 "macro_external_10": {
  "balanced_acc": 0.8481,
  "auroc": 0.8645,
  "tpr@1%fpr": 0.3084,
  "fpr@0.5": 0.1298,
  "ece": 0.1315
 },
 "capabilities": {
  "Prompt injection (direct + indirect)": {
   "benchmarks": 14,
   "balanced_acc": 0.952,
   "auroc": 0.9905,
   "tpr@1%fpr": 0.9208,
   "fpr@0.5": 0.0179,
   "ece": 0.0412
  },
  "Agent / tool security": {
   "benchmarks": 6,
   "balanced_acc": 0.9632,
   "auroc": 0.9885,
   "tpr@1%fpr": 0.9293,
   "fpr@0.5": 0.0122,
   "ece": 0.028
  },
  "Sensitive data / PII": {
   "benchmarks": 4,
   "balanced_acc": 0.9802,
   "auroc": 0.9947,
   "tpr@1%fpr": 0.9675,
   "fpr@0.5": 0.0046,
   "ece": 0.0076
  },
  "Multilingual / non-English": {
   "benchmarks": 8,
   "balanced_acc": 0.8975,
   "auroc": 0.959,
   "tpr@1%fpr": 0.7535,
   "fpr@0.5": 0.033,
   "ece": 0.0813
  },
  "Jailbreak / unsafe requests": {
   "benchmarks": 6,
   "balanced_acc": 0.8335,
   "auroc": 0.8881,
   "tpr@1%fpr": 0.3604,
   "fpr@0.5": 0.1712,
   "ece": 0.1407
  },
  "Hard negatives / over-defense (benign-only; FPR)": {
   "benchmarks": 6,
   "balanced_acc": 0.944,
   "auroc": 0.8346,
   "tpr@1%fpr": 0.18,
   "fpr@0.5": 0.0711,
   "ece": 0.0487
  }
 },
 "benchmarks": {
  "aegis2-content-safety-test": {
   "name": "Aegis 2.0 (test)",
   "decision": "harmful.request",
   "n": 1750,
   "external": false,
   "balanced_acc": 0.8445,
   "auroc": 0.9168,
   "tpr@1%fpr": 0.3709,
   "fpr@0.5": 0.1683,
   "ece": 0.1121
  },
  "agentic-pi-5k-test": {
   "name": "agentic-PI-5k (test)",
   "decision": "injection.presence",
   "n": 1365,
   "external": false,
   "balanced_acc": 0.9829,
   "auroc": 0.9995,
   "tpr@1%fpr": 0.9857,
   "fpr@0.5": 0.0377,
   "ece": 0.0176
  },
  "arabic-guardrail-hard": {
   "name": "Arabic guardrail (hard)",
   "decision": "harmful.request",
   "n": 1456,
   "external": false,
   "balanced_acc": 0.9808,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": 0.0192,
   "ece": 0.0138
  },
  "boundary-pairs-test": {
   "name": "boundary pairs (test)",
   "decision": "injection.presence",
   "n": 879,
   "external": false,
   "balanced_acc": 1.0,
   "auroc": 1.0,
   "tpr@1%fpr": 1.0,
   "fpr@0.5": 0.0,
   "ece": 0.0
  },
  "browsesafe-test": {
   "name": "BrowseSafe (test)",
   "decision": "injection.presence",
   "n": 8227,
   "external": false,
   "balanced_acc": 0.8273,
   "auroc": 0.9343,
   "tpr@1%fpr": 0.6393,
   "fpr@0.5": 0.0329,
   "ece": 0.129
  },
  "deepset-prompt-injections-multilingual-test": {
   "name": "deepset prompt-injections multilingual (test)",
   "decision": "injection.presence",
   "n": 1322,
   "external": false,
   "balanced_acc": 0.9846,
   "auroc": 0.9999,
   "tpr@1%fpr": 1.0,
   "fpr@0.5": 0.0,
   "ece": 0.0076
  },
  "deepset-prompt-injections-test": {
   "name": "deepset/prompt-injections (test)",
   "decision": "injection.presence",
   "n": 182,
   "external": false,
   "balanced_acc": 1.0,
   "auroc": 1.0,
   "tpr@1%fpr": 1.0,
   "fpr@0.5": 0.0,
   "ece": 0.0026
  },
  "gretel-pii-finance-multilingual-test": {
   "name": "Gretel PII finance multilingual (test)",
   "decision": "secret.presence",
   "n": 3566,
   "external": false,
   "balanced_acc": 0.9341,
   "auroc": 0.9805,
   "tpr@1%fpr": 0.8812,
   "fpr@0.5": 0.0129,
   "ece": 0.0192
  },
  "gretel-pii-masking-en-test": {
   "name": "Gretel PII masking (test)",
   "decision": "secret.presence",
   "n": 3512,
   "external": false,
   "balanced_acc": 0.9979,
   "auroc": 1.0,
   "tpr@1%fpr": 1.0,
   "fpr@0.5": 0.0041,
   "ece": 0.0045
  },
  "jackhhao-jailbreak-classification-test": {
   "name": "jackhhao jailbreak-classification (test)",
   "decision": "jailbreak.presence",
   "n": 800,
   "external": true,
   "balanced_acc": 0.9129,
   "auroc": 0.9511,
   "tpr@1%fpr": 0.5887,
   "fpr@0.5": 0.0772,
   "ece": 0.0859
  },
  "jbb-behaviors": {
   "name": "JailbreakBench behaviors",
   "decision": "harmful.request",
   "n": 200,
   "external": true,
   "balanced_acc": 0.745,
   "auroc": 0.8404,
   "tpr@1%fpr": 0.29,
   "fpr@0.5": 0.34,
   "ece": 0.2075
  },
  "jpi-ja": {
   "name": "Japanese indirect injection (10 probe sets, mean)",
   "decision": "injection.presence",
   "n": 180,
   "external": true,
   "balanced_acc": 0.6817,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": null,
   "ece": 0.2903
  },
  "lakera-gandalf-ignore-instructions-test": {
   "name": "Lakera Gandalf ignore-instructions (test)",
   "decision": "injection.presence",
   "n": 328,
   "external": false,
   "balanced_acc": 1.0,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": null,
   "ece": 0.0
  },
  "lakera-gandalf-summarization-test": {
   "name": "Lakera Gandalf summarization (test)",
   "decision": "injection.presence",
   "n": 30,
   "external": false,
   "balanced_acc": 1.0,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": null,
   "ece": 0.0
  },
  "lakera-mosscap-test": {
   "name": "Lakera Mosscap (test)",
   "decision": "prompt_leak.attempt",
   "n": 6767,
   "external": false,
   "balanced_acc": 1.0,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": null,
   "ece": 0.0
  },
  "massive-benign-test": {
   "name": "MASSIVE benign (FP control)",
   "decision": "injection.presence",
   "n": 13042,
   "external": false,
   "balanced_acc": 1.0,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": 0.0,
   "ece": 0.0
  },
  "nemotron-pii-test": {
   "name": "Nemotron PII (test)",
   "decision": "secret.presence",
   "n": 7242,
   "external": false,
   "balanced_acc": 0.9921,
   "auroc": 0.9984,
   "tpr@1%fpr": 0.9909,
   "fpr@0.5": 0.0014,
   "ece": 0.0039
  },
  "nemotron-safety-guard-v3-test": {
   "name": "Nemotron Safety Guard v3 (test)",
   "decision": "harmful.request",
   "n": 6914,
   "external": false,
   "balanced_acc": 0.7505,
   "auroc": 0.8442,
   "tpr@1%fpr": 0.1919,
   "fpr@0.5": 0.0993,
   "ece": 0.2029
  },
  "notinject-notinject-one": {
   "name": "NotInject-1 (over-defense)",
   "decision": "injection.presence",
   "n": 226,
   "external": true,
   "balanced_acc": 0.9823,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": 0.0177,
   "ece": 0.0144
  },
  "notinject-notinject-three": {
   "name": "NotInject-3 (over-defense)",
   "decision": "injection.presence",
   "n": 226,
   "external": true,
   "balanced_acc": 0.9646,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": 0.0354,
   "ece": 0.039
  },
  "notinject-notinject-two": {
   "name": "NotInject-2 (over-defense)",
   "decision": "injection.presence",
   "n": 226,
   "external": true,
   "balanced_acc": 0.9912,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": 0.0088,
   "ece": 0.0076
  },
  "or-bench-hard": {
   "name": "OR-Bench hard (over-refusal)",
   "decision": "harmful.request",
   "n": 1319,
   "external": false,
   "balanced_acc": 0.9871,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": 0.0129,
   "ece": 0.0074
  },
  "patronus-holdout-llmail-holdout": {
   "name": "Patronus holdout: llmail-holdout",
   "decision": "injection.presence",
   "n": 5721,
   "external": false,
   "balanced_acc": 0.9997,
   "auroc": 1.0,
   "tpr@1%fpr": 1.0,
   "fpr@0.5": 0.0,
   "ece": 0.0008
  },
  "patronus-holdout-mosscap-unseen-lang": {
   "name": "Patronus holdout: mosscap-unseen-lang",
   "decision": "prompt_leak.attempt",
   "n": 22,
   "external": false,
   "balanced_acc": 1.0,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": null,
   "ece": 0.0
  },
  "patronus-holdout-synth-iid": {
   "name": "Patronus holdout: synth-iid",
   "decision": "injection.presence",
   "n": 19494,
   "external": false,
   "balanced_acc": 0.969,
   "auroc": 0.9975,
   "tpr@1%fpr": 0.9509,
   "fpr@0.5": 0.0026,
   "ece": 0.0202
  },
  "patronus-holdout-synth-toolpoison-probe": {
   "name": "Patronus holdout: synth-toolpoison-probe",
   "decision": "tool_metadata.poisoning",
   "n": 2509,
   "external": false,
   "balanced_acc": 1.0,
   "auroc": 1.0,
   "tpr@1%fpr": 1.0,
   "fpr@0.5": 0.0,
   "ece": 0.0
  },
  "polyguard-prompts-test": {
   "name": "PolyGuard prompts (multilingual, test)",
   "decision": "harmful.request",
   "n": 64170,
   "external": true,
   "balanced_acc": 0.7157,
   "auroc": 0.8318,
   "tpr@1%fpr": 0.1749,
   "fpr@0.5": 0.0778,
   "ece": 0.211
  },
  "privy-pii-test": {
   "name": "Privy PII (test)",
   "decision": "sensitive.presence",
   "n": 2383,
   "external": false,
   "balanced_acc": 0.9968,
   "auroc": 0.9999,
   "tpr@1%fpr": 0.9978,
   "fpr@0.5": 0.0,
   "ece": 0.0029
  },
  "simsonsun-jailbreak-prompts-dataset-1": {
   "name": "simsonsun jailbreak prompts 1",
   "decision": "jailbreak.presence",
   "n": 67,
   "external": true,
   "balanced_acc": 0.8612,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": null,
   "ece": 0.1411
  },
  "simsonsun-jailbreak-prompts-dataset-2": {
   "name": "simsonsun jailbreak prompts 2",
   "decision": "jailbreak.presence",
   "n": 2284,
   "external": true,
   "balanced_acc": 0.8871,
   "auroc": null,
   "tpr@1%fpr": null,
   "fpr@0.5": null,
   "ece": 0.0945
  },
  "turkish-conversation-pi-test": {
   "name": "Turkish conversation PI (test)",
   "decision": "injection.presence",
   "n": 243,
   "external": false,
   "balanced_acc": 0.9167,
   "auroc": 0.9907,
   "tpr@1%fpr": 0.8,
   "fpr@0.5": 0.0,
   "ece": 0.0632
  },
  "turkish-pi-1k-test": {
   "name": "Turkish PI 1k (test)",
   "decision": "injection.presence",
   "n": 246,
   "external": false,
   "balanced_acc": 0.9664,
   "auroc": 0.9922,
   "tpr@1%fpr": 0.9114,
   "fpr@0.5": 0.0882,
   "ece": 0.0452
  },
  "xstest": {
   "name": "XSTest (over-refusal)",
   "decision": "harmful.request",
   "n": 450,
   "external": true,
   "balanced_acc": 0.739,
   "auroc": 0.8346,
   "tpr@1%fpr": 0.18,
   "fpr@0.5": 0.352,
   "ece": 0.224
  }
 }
}