Text Classification
PyTorch
ONNX
Safetensors
decision-model
jev-like
systemone
prompt-injection
indirect-prompt-injection
ai-agent-security
tool-security
jailbreak-detection
data-loss-prevention
guardrails
calibration
multiple-choice
structured-output
classification
security
llm-security
ai-safety
multilingual
modernbert
mmbert
cpu
patronus
wolf-defender-jev
decision-model-guard
context-aware-security
decision-security
agent-security
ai-security
agent-action-security
prompt-injection-detection
mcp-security
untrusted-content
local-inference
on-device
cpu-inference
onnxruntime
quantized
int8
int4
multiple-choice-classification
jev
decision-steering
agent-guardrails
tool-calling
policy-evaluation
sensitive-data-detection
pii-detection
Eval Results (legacy)
| { | |
| "model": "Wolf Defender Jev", | |
| "description": "33 held-out benchmarks, primary decision each; the 10 Japanese indirect-injection probe sets count as one. 'external' = source not seen during training.", | |
| "macro_all_33": { | |
| "balanced_acc": 0.9276, | |
| "auroc": 0.9556, | |
| "tpr@1%fpr": 0.7477, | |
| "fpr@0.5": 0.0534, | |
| "ece": 0.0596 | |
| }, | |
| "macro_external_10": { | |
| "balanced_acc": 0.8481, | |
| "auroc": 0.8645, | |
| "tpr@1%fpr": 0.3084, | |
| "fpr@0.5": 0.1298, | |
| "ece": 0.1315 | |
| }, | |
| "capabilities": { | |
| "Prompt injection (direct + indirect)": { | |
| "benchmarks": 14, | |
| "balanced_acc": 0.952, | |
| "auroc": 0.9905, | |
| "tpr@1%fpr": 0.9208, | |
| "fpr@0.5": 0.0179, | |
| "ece": 0.0412 | |
| }, | |
| "Agent / tool security": { | |
| "benchmarks": 6, | |
| "balanced_acc": 0.9632, | |
| "auroc": 0.9885, | |
| "tpr@1%fpr": 0.9293, | |
| "fpr@0.5": 0.0122, | |
| "ece": 0.028 | |
| }, | |
| "Sensitive data / PII": { | |
| "benchmarks": 4, | |
| "balanced_acc": 0.9802, | |
| "auroc": 0.9947, | |
| "tpr@1%fpr": 0.9675, | |
| "fpr@0.5": 0.0046, | |
| "ece": 0.0076 | |
| }, | |
| "Multilingual / non-English": { | |
| "benchmarks": 8, | |
| "balanced_acc": 0.8975, | |
| "auroc": 0.959, | |
| "tpr@1%fpr": 0.7535, | |
| "fpr@0.5": 0.033, | |
| "ece": 0.0813 | |
| }, | |
| "Jailbreak / unsafe requests": { | |
| "benchmarks": 6, | |
| "balanced_acc": 0.8335, | |
| "auroc": 0.8881, | |
| "tpr@1%fpr": 0.3604, | |
| "fpr@0.5": 0.1712, | |
| "ece": 0.1407 | |
| }, | |
| "Hard negatives / over-defense (benign-only; FPR)": { | |
| "benchmarks": 6, | |
| "balanced_acc": 0.944, | |
| "auroc": 0.8346, | |
| "tpr@1%fpr": 0.18, | |
| "fpr@0.5": 0.0711, | |
| "ece": 0.0487 | |
| } | |
| }, | |
| "benchmarks": { | |
| "aegis2-content-safety-test": { | |
| "name": "Aegis 2.0 (test)", | |
| "decision": "harmful.request", | |
| "n": 1750, | |
| "external": false, | |
| "balanced_acc": 0.8445, | |
| "auroc": 0.9168, | |
| "tpr@1%fpr": 0.3709, | |
| "fpr@0.5": 0.1683, | |
| "ece": 0.1121 | |
| }, | |
| "agentic-pi-5k-test": { | |
| "name": "agentic-PI-5k (test)", | |
| "decision": "injection.presence", | |
| "n": 1365, | |
| "external": false, | |
| "balanced_acc": 0.9829, | |
| "auroc": 0.9995, | |
| "tpr@1%fpr": 0.9857, | |
| "fpr@0.5": 0.0377, | |
| "ece": 0.0176 | |
| }, | |
| "arabic-guardrail-hard": { | |
| "name": "Arabic guardrail (hard)", | |
| "decision": "harmful.request", | |
| "n": 1456, | |
| "external": false, | |
| "balanced_acc": 0.9808, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": 0.0192, | |
| "ece": 0.0138 | |
| }, | |
| "boundary-pairs-test": { | |
| "name": "boundary pairs (test)", | |
| "decision": "injection.presence", | |
| "n": 879, | |
| "external": false, | |
| "balanced_acc": 1.0, | |
| "auroc": 1.0, | |
| "tpr@1%fpr": 1.0, | |
| "fpr@0.5": 0.0, | |
| "ece": 0.0 | |
| }, | |
| "browsesafe-test": { | |
| "name": "BrowseSafe (test)", | |
| "decision": "injection.presence", | |
| "n": 8227, | |
| "external": false, | |
| "balanced_acc": 0.8273, | |
| "auroc": 0.9343, | |
| "tpr@1%fpr": 0.6393, | |
| "fpr@0.5": 0.0329, | |
| "ece": 0.129 | |
| }, | |
| "deepset-prompt-injections-multilingual-test": { | |
| "name": "deepset prompt-injections multilingual (test)", | |
| "decision": "injection.presence", | |
| "n": 1322, | |
| "external": false, | |
| "balanced_acc": 0.9846, | |
| "auroc": 0.9999, | |
| "tpr@1%fpr": 1.0, | |
| "fpr@0.5": 0.0, | |
| "ece": 0.0076 | |
| }, | |
| "deepset-prompt-injections-test": { | |
| "name": "deepset/prompt-injections (test)", | |
| "decision": "injection.presence", | |
| "n": 182, | |
| "external": false, | |
| "balanced_acc": 1.0, | |
| "auroc": 1.0, | |
| "tpr@1%fpr": 1.0, | |
| "fpr@0.5": 0.0, | |
| "ece": 0.0026 | |
| }, | |
| "gretel-pii-finance-multilingual-test": { | |
| "name": "Gretel PII finance multilingual (test)", | |
| "decision": "secret.presence", | |
| "n": 3566, | |
| "external": false, | |
| "balanced_acc": 0.9341, | |
| "auroc": 0.9805, | |
| "tpr@1%fpr": 0.8812, | |
| "fpr@0.5": 0.0129, | |
| "ece": 0.0192 | |
| }, | |
| "gretel-pii-masking-en-test": { | |
| "name": "Gretel PII masking (test)", | |
| "decision": "secret.presence", | |
| "n": 3512, | |
| "external": false, | |
| "balanced_acc": 0.9979, | |
| "auroc": 1.0, | |
| "tpr@1%fpr": 1.0, | |
| "fpr@0.5": 0.0041, | |
| "ece": 0.0045 | |
| }, | |
| "jackhhao-jailbreak-classification-test": { | |
| "name": "jackhhao jailbreak-classification (test)", | |
| "decision": "jailbreak.presence", | |
| "n": 800, | |
| "external": true, | |
| "balanced_acc": 0.9129, | |
| "auroc": 0.9511, | |
| "tpr@1%fpr": 0.5887, | |
| "fpr@0.5": 0.0772, | |
| "ece": 0.0859 | |
| }, | |
| "jbb-behaviors": { | |
| "name": "JailbreakBench behaviors", | |
| "decision": "harmful.request", | |
| "n": 200, | |
| "external": true, | |
| "balanced_acc": 0.745, | |
| "auroc": 0.8404, | |
| "tpr@1%fpr": 0.29, | |
| "fpr@0.5": 0.34, | |
| "ece": 0.2075 | |
| }, | |
| "jpi-ja": { | |
| "name": "Japanese indirect injection (10 probe sets, mean)", | |
| "decision": "injection.presence", | |
| "n": 180, | |
| "external": true, | |
| "balanced_acc": 0.6817, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": null, | |
| "ece": 0.2903 | |
| }, | |
| "lakera-gandalf-ignore-instructions-test": { | |
| "name": "Lakera Gandalf ignore-instructions (test)", | |
| "decision": "injection.presence", | |
| "n": 328, | |
| "external": false, | |
| "balanced_acc": 1.0, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": null, | |
| "ece": 0.0 | |
| }, | |
| "lakera-gandalf-summarization-test": { | |
| "name": "Lakera Gandalf summarization (test)", | |
| "decision": "injection.presence", | |
| "n": 30, | |
| "external": false, | |
| "balanced_acc": 1.0, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": null, | |
| "ece": 0.0 | |
| }, | |
| "lakera-mosscap-test": { | |
| "name": "Lakera Mosscap (test)", | |
| "decision": "prompt_leak.attempt", | |
| "n": 6767, | |
| "external": false, | |
| "balanced_acc": 1.0, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": null, | |
| "ece": 0.0 | |
| }, | |
| "massive-benign-test": { | |
| "name": "MASSIVE benign (FP control)", | |
| "decision": "injection.presence", | |
| "n": 13042, | |
| "external": false, | |
| "balanced_acc": 1.0, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": 0.0, | |
| "ece": 0.0 | |
| }, | |
| "nemotron-pii-test": { | |
| "name": "Nemotron PII (test)", | |
| "decision": "secret.presence", | |
| "n": 7242, | |
| "external": false, | |
| "balanced_acc": 0.9921, | |
| "auroc": 0.9984, | |
| "tpr@1%fpr": 0.9909, | |
| "fpr@0.5": 0.0014, | |
| "ece": 0.0039 | |
| }, | |
| "nemotron-safety-guard-v3-test": { | |
| "name": "Nemotron Safety Guard v3 (test)", | |
| "decision": "harmful.request", | |
| "n": 6914, | |
| "external": false, | |
| "balanced_acc": 0.7505, | |
| "auroc": 0.8442, | |
| "tpr@1%fpr": 0.1919, | |
| "fpr@0.5": 0.0993, | |
| "ece": 0.2029 | |
| }, | |
| "notinject-notinject-one": { | |
| "name": "NotInject-1 (over-defense)", | |
| "decision": "injection.presence", | |
| "n": 226, | |
| "external": true, | |
| "balanced_acc": 0.9823, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": 0.0177, | |
| "ece": 0.0144 | |
| }, | |
| "notinject-notinject-three": { | |
| "name": "NotInject-3 (over-defense)", | |
| "decision": "injection.presence", | |
| "n": 226, | |
| "external": true, | |
| "balanced_acc": 0.9646, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": 0.0354, | |
| "ece": 0.039 | |
| }, | |
| "notinject-notinject-two": { | |
| "name": "NotInject-2 (over-defense)", | |
| "decision": "injection.presence", | |
| "n": 226, | |
| "external": true, | |
| "balanced_acc": 0.9912, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": 0.0088, | |
| "ece": 0.0076 | |
| }, | |
| "or-bench-hard": { | |
| "name": "OR-Bench hard (over-refusal)", | |
| "decision": "harmful.request", | |
| "n": 1319, | |
| "external": false, | |
| "balanced_acc": 0.9871, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": 0.0129, | |
| "ece": 0.0074 | |
| }, | |
| "patronus-holdout-llmail-holdout": { | |
| "name": "Patronus holdout: llmail-holdout", | |
| "decision": "injection.presence", | |
| "n": 5721, | |
| "external": false, | |
| "balanced_acc": 0.9997, | |
| "auroc": 1.0, | |
| "tpr@1%fpr": 1.0, | |
| "fpr@0.5": 0.0, | |
| "ece": 0.0008 | |
| }, | |
| "patronus-holdout-mosscap-unseen-lang": { | |
| "name": "Patronus holdout: mosscap-unseen-lang", | |
| "decision": "prompt_leak.attempt", | |
| "n": 22, | |
| "external": false, | |
| "balanced_acc": 1.0, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": null, | |
| "ece": 0.0 | |
| }, | |
| "patronus-holdout-synth-iid": { | |
| "name": "Patronus holdout: synth-iid", | |
| "decision": "injection.presence", | |
| "n": 19494, | |
| "external": false, | |
| "balanced_acc": 0.969, | |
| "auroc": 0.9975, | |
| "tpr@1%fpr": 0.9509, | |
| "fpr@0.5": 0.0026, | |
| "ece": 0.0202 | |
| }, | |
| "patronus-holdout-synth-toolpoison-probe": { | |
| "name": "Patronus holdout: synth-toolpoison-probe", | |
| "decision": "tool_metadata.poisoning", | |
| "n": 2509, | |
| "external": false, | |
| "balanced_acc": 1.0, | |
| "auroc": 1.0, | |
| "tpr@1%fpr": 1.0, | |
| "fpr@0.5": 0.0, | |
| "ece": 0.0 | |
| }, | |
| "polyguard-prompts-test": { | |
| "name": "PolyGuard prompts (multilingual, test)", | |
| "decision": "harmful.request", | |
| "n": 64170, | |
| "external": true, | |
| "balanced_acc": 0.7157, | |
| "auroc": 0.8318, | |
| "tpr@1%fpr": 0.1749, | |
| "fpr@0.5": 0.0778, | |
| "ece": 0.211 | |
| }, | |
| "privy-pii-test": { | |
| "name": "Privy PII (test)", | |
| "decision": "sensitive.presence", | |
| "n": 2383, | |
| "external": false, | |
| "balanced_acc": 0.9968, | |
| "auroc": 0.9999, | |
| "tpr@1%fpr": 0.9978, | |
| "fpr@0.5": 0.0, | |
| "ece": 0.0029 | |
| }, | |
| "simsonsun-jailbreak-prompts-dataset-1": { | |
| "name": "simsonsun jailbreak prompts 1", | |
| "decision": "jailbreak.presence", | |
| "n": 67, | |
| "external": true, | |
| "balanced_acc": 0.8612, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": null, | |
| "ece": 0.1411 | |
| }, | |
| "simsonsun-jailbreak-prompts-dataset-2": { | |
| "name": "simsonsun jailbreak prompts 2", | |
| "decision": "jailbreak.presence", | |
| "n": 2284, | |
| "external": true, | |
| "balanced_acc": 0.8871, | |
| "auroc": null, | |
| "tpr@1%fpr": null, | |
| "fpr@0.5": null, | |
| "ece": 0.0945 | |
| }, | |
| "turkish-conversation-pi-test": { | |
| "name": "Turkish conversation PI (test)", | |
| "decision": "injection.presence", | |
| "n": 243, | |
| "external": false, | |
| "balanced_acc": 0.9167, | |
| "auroc": 0.9907, | |
| "tpr@1%fpr": 0.8, | |
| "fpr@0.5": 0.0, | |
| "ece": 0.0632 | |
| }, | |
| "turkish-pi-1k-test": { | |
| "name": "Turkish PI 1k (test)", | |
| "decision": "injection.presence", | |
| "n": 246, | |
| "external": false, | |
| "balanced_acc": 0.9664, | |
| "auroc": 0.9922, | |
| "tpr@1%fpr": 0.9114, | |
| "fpr@0.5": 0.0882, | |
| "ece": 0.0452 | |
| }, | |
| "xstest": { | |
| "name": "XSTest (over-refusal)", | |
| "decision": "harmful.request", | |
| "n": 450, | |
| "external": true, | |
| "balanced_acc": 0.739, | |
| "auroc": 0.8346, | |
| "tpr@1%fpr": 0.18, | |
| "fpr@0.5": 0.352, | |
| "ece": 0.224 | |
| } | |
| } | |
| } |