| { |
| "type": "aggregate_prompt_injection_benchmarks", |
| "models": [ |
| { |
| "model": "TextCortex/clef-cybersecurity", |
| "name": "clef-cybersecurity", |
| "full_en": { |
| "n": 510, |
| "attacks": 256, |
| "clean": 254, |
| "auroc": 0.9925335260826772, |
| "caught": 232, |
| "false_alarms": 0 |
| }, |
| "full_de": { |
| "n": 510, |
| "attacks": 256, |
| "clean": 254, |
| "auroc": 0.9743940698818898, |
| "caught": 241, |
| "false_alarms": 17 |
| }, |
| "skills_en": { |
| "n": 48, |
| "attacks": 27, |
| "clean": 21, |
| "auroc": 1.0, |
| "caught": 23, |
| "false_alarms": 0 |
| }, |
| "skills_de": { |
| "n": 48, |
| "attacks": 27, |
| "clean": 21, |
| "auroc": 0.9171075837742504, |
| "caught": 26, |
| "false_alarms": 3 |
| }, |
| "pdf": { |
| "n": 730, |
| "attacks": 107, |
| "clean": 623, |
| "auroc": 0.9856212778086137, |
| "caught": 84, |
| "false_alarms": 3 |
| }, |
| "decision_rule": "score > 0.5" |
| }, |
| { |
| "model": "jev", |
| "name": "Jev", |
| "full_en": { |
| "auroc": 0.9799920029527559, |
| "n": 510 |
| }, |
| "full_de": { |
| "auroc": 0.9564391609251969, |
| "n": 510 |
| }, |
| "skills_en": { |
| "n": 48, |
| "attacks": 27, |
| "clean": 21, |
| "auroc": 0.9841269841269841, |
| "caught": 25, |
| "false_alarms": 1 |
| }, |
| "skills_de": { |
| "n": 48, |
| "attacks": 27, |
| "clean": 21, |
| "auroc": 0.9603174603174603, |
| "caught": 25, |
| "false_alarms": 2 |
| }, |
| "pdf": { |
| "n": 730, |
| "attacks": 107, |
| "clean": 623, |
| "auroc": 0.978525674682348, |
| "caught": 73, |
| "false_alarms": 2 |
| }, |
| "decision_rule": "score > 0.5" |
| }, |
| { |
| "model": "laya-cybersec-r2a", |
| "name": "Laya R2a", |
| "full_en": { |
| "n": 510, |
| "attacks": 256, |
| "clean": 254, |
| "auroc": 0.9155004306102362, |
| "caught": 140, |
| "false_alarms": 2, |
| "recall": 0.546875, |
| "fpr": 0.007874015748031496 |
| }, |
| "full_de": { |
| "n": 510, |
| "attacks": 256, |
| "clean": 254, |
| "auroc": 0.8780065821850394, |
| "caught": 151, |
| "false_alarms": 9, |
| "recall": 0.58984375, |
| "fpr": 0.03543307086614173 |
| }, |
| "skills_en": { |
| "n": 48, |
| "attacks": 27, |
| "clean": 21, |
| "auroc": 0.927689594356261, |
| "caught": 18, |
| "false_alarms": 0 |
| }, |
| "skills_de": { |
| "n": 48, |
| "attacks": 27, |
| "clean": 21, |
| "auroc": 0.9329805996472663, |
| "caught": 18, |
| "false_alarms": 0 |
| }, |
| "pdf": { |
| "n": 730, |
| "attacks": 107, |
| "clean": 623, |
| "auroc": 0.8855927753859077, |
| "caught": 81, |
| "false_alarms": 0 |
| }, |
| "decision_rule": "score > 0.95" |
| } |
| ], |
| "cohort": { |
| "full_per_language": 510, |
| "skills_per_language": 48, |
| "pdf_attacks": 107, |
| "clean_pdfs": 623 |
| }, |
| "scope": "Previously inspected internal regression sets; not a public leaderboard or fresh blind test.", |
| "threshold_note": "Strict >0.5 for CLEF and Jev; strict >0.95 for Laya R2a. Recall and false alarms are not equal-threshold comparisons.", |
| "selected_epoch": 3, |
| "epochs_trained": 4, |
| "clef_secondary_pdf_at_095": { |
| "n": 730, |
| "attacks": 107, |
| "clean": 623, |
| "auroc": 0.9856212778086137, |
| "caught": 69, |
| "false_alarms": 2 |
| } |
| } |
|
|
