Hugging Face's logo

patronus-studio
/
cat-shell-risk-commander

Text Classification
Scikit-learn
shell-security
shell-risk
bash
powershell
cmd
coding-agents
tf-idf
logistic-regression
cpu
research
cat-shell-risk-commander
shell-command-classification
shell-command-security
shell-risk-classification
command-risk-detection
agent-security
ai-agent-security
ai-security
tool-call-security
tool-security
posix
windows
linux
powershell-security
tfidf
shell-token-features
binary-classification
local-inference
on-device
cpu-inference
scikit-learn
json-model
a1
Eval Results (legacy)
Model card Files
xet
 Community
cat-shell-risk-commander / evaluation.json
patronus-protect's picture
patronus-protect
A1 + relations + heredoc masking (run D)
92aae0e verified
Raw History Blame Contribute Delete
17.3 kB
{
"operating_points": {
"external_commands_1pct_fpr": 0.364353,
"block_0_1pct_fpr": 2.889838
},
"default_policy": "external_commands_1pct_fpr",
"overall_test": {
"external_commands_1pct_fpr": {
"n": 6902,
"pos": 1080,
"tp": 1020,
"fp": 43,
"fn": 60,
"tn": 5779,
"recall": 0.9444,
"fpr": 0.0074,
"fpr_wilson_upper": 0.0099,
"ap": 0.9892,
"auc": 0.9977
},
"block_0_1pct_fpr": {
"n": 6902,
"pos": 1080,
"tp": 808,
"fp": 4,
"fn": 272,
"tn": 5818,
"recall": 0.7481,
"fpr": 0.0007,
"fpr_wilson_upper": 0.0018,
"ap": 0.9892,
"auc": 0.9977
}
},
"original_bash_test": {
"external_commands_1pct_fpr": {
"n": 4194,
"pos": 193,
"tp": 167,
"fp": 13,
"fn": 26,
"tn": 3988,
"recall": 0.8653,
"fpr": 0.0032,
"fpr_wilson_upper": 0.0056,
"ap": 0.9646,
"auc": 0.9971
},
"block_0_1pct_fpr": {
"n": 4194,
"pos": 193,
"tp": 123,
"fp": 1,
"fn": 70,
"tn": 4000,
"recall": 0.6373,
"fpr": 0.0002,
"fpr_wilson_upper": 0.0014,
"ap": 0.9646,
"auc": 0.9971
}
},
"per_family": {
"shellsafety_powershell": {
"external_commands_1pct_fpr": {
"n": 119,
"pos": 58,
"tp": 58,
"fp": 0,
"fn": 0,
"tn": 61,
"recall": 1.0,
"fpr": 0.0,
"fpr_wilson_upper": 0.0592,
"ap": 1.0,
"auc": 1.0
},
"block_0_1pct_fpr": {
"n": 119,
"pos": 58,
"tp": 48,
"fp": 0,
"fn": 10,
"tn": 61,
"recall": 0.8276,
"fpr": 0.0,
"fpr_wilson_upper": 0.0592,
"ap": 1.0,
"auc": 1.0
}
},
"shellsafety_posix": {
"external_commands_1pct_fpr": {
"n": 735,
"pos": 292,
"tp": 271,
"fp": 9,
"fn": 21,
"tn": 434,
"recall": 0.9281,
"fpr": 0.0203,
"fpr_wilson_upper": 0.0382,
"ap": 0.9895,
"auc": 0.9921
},
"block_0_1pct_fpr": {
"n": 735,
"pos": 292,
"tp": 204,
"fp": 0,
"fn": 88,
"tn": 443,
"recall": 0.6986,
"fpr": 0.0,
"fpr_wilson_upper": 0.0086,
"ap": 0.9895,
"auc": 0.9921
}
},
"shellsafety_cmd": {
"external_commands_1pct_fpr": {
"n": 64,
"pos": 24,
"tp": 23,
"fp": 0,
"fn": 1,
"tn": 40,
"recall": 0.9583,
"fpr": 0.0,
"fpr_wilson_upper": 0.0876,
"ap": 1.0,
"auc": 1.0
},
"block_0_1pct_fpr": {
"n": 64,
"pos": 24,
"tp": 16,
"fp": 0,
"fn": 8,
"tn": 40,
"recall": 0.6667,
"fpr": 0.0,
"fpr_wilson_upper": 0.0876,
"ap": 1.0,
"auc": 1.0
}
},
"original_bash": {
"external_commands_1pct_fpr": {
"n": 4194,
"pos": 193,
"tp": 167,
"fp": 13,
"fn": 26,
"tn": 3988,
"recall": 0.8653,
"fpr": 0.0032,
"fpr_wilson_upper": 0.0056,
"ap": 0.9646,
"auc": 0.9971
},
"block_0_1pct_fpr": {
"n": 4194,
"pos": 193,
"tp": 123,
"fp": 1,
"fn": 70,
"tn": 4000,
"recall": 0.6373,
"fpr": 0.0002,
"fpr_wilson_upper": 0.0014,
"ap": 0.9646,
"auc": 0.9971
}
},
"exfil_pipe": {
"external_commands_1pct_fpr": {
"n": 160,
"pos": 160,
"tp": 154,
"fp": 0,
"fn": 6,
"tn": 0,
"recall": 0.9625,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 160,
"pos": 160,
"tp": 143,
"fp": 0,
"fn": 17,
"tn": 0,
"recall": 0.8938,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"credential_store": {
"external_commands_1pct_fpr": {
"n": 80,
"pos": 80,
"tp": 80,
"fp": 0,
"fn": 0,
"tn": 0,
"recall": 1.0,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 80,
"pos": 80,
"tp": 67,
"fp": 0,
"fn": 13,
"tn": 0,
"recall": 0.8375,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"reverse_shell": {
"external_commands_1pct_fpr": {
"n": 7,
"pos": 7,
"tp": 6,
"fp": 0,
"fn": 1,
"tn": 0,
"recall": 0.8571,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 7,
"pos": 7,
"tp": 3,
"fp": 0,
"fn": 4,
"tn": 0,
"recall": 0.4286,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"remote_download_exec": {
"external_commands_1pct_fpr": {
"n": 147,
"pos": 147,
"tp": 145,
"fp": 0,
"fn": 2,
"tn": 0,
"recall": 0.9864,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 147,
"pos": 147,
"tp": 135,
"fp": 0,
"fn": 12,
"tn": 0,
"recall": 0.9184,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"privilege_escalation": {
"external_commands_1pct_fpr": {
"n": 8,
"pos": 8,
"tp": 8,
"fp": 0,
"fn": 0,
"tn": 0,
"recall": 1.0,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 8,
"pos": 8,
"tp": 6,
"fp": 0,
"fn": 2,
"tn": 0,
"recall": 0.75,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"defense_evasion": {
"external_commands_1pct_fpr": {
"n": 26,
"pos": 26,
"tp": 26,
"fp": 0,
"fn": 0,
"tn": 0,
"recall": 1.0,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 26,
"pos": 26,
"tp": 17,
"fp": 0,
"fn": 9,
"tn": 0,
"recall": 0.6538,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"disk_destruction": {
"external_commands_1pct_fpr": {
"n": 28,
"pos": 28,
"tp": 28,
"fp": 0,
"fn": 0,
"tn": 0,
"recall": 1.0,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 28,
"pos": 28,
"tp": 16,
"fp": 0,
"fn": 12,
"tn": 0,
"recall": 0.5714,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"resource_dos": {
"external_commands_1pct_fpr": {
"n": 4,
"pos": 4,
"tp": 3,
"fp": 0,
"fn": 1,
"tn": 0,
"recall": 0.75,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 4,
"pos": 4,
"tp": 0,
"fp": 0,
"fn": 4,
"tn": 0,
"recall": 0.0,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"payload_decode_exec": {
"external_commands_1pct_fpr": {
"n": 6,
"pos": 6,
"tp": 6,
"fp": 0,
"fn": 0,
"tn": 0,
"recall": 1.0,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 6,
"pos": 6,
"tp": 5,
"fp": 0,
"fn": 1,
"tn": 0,
"recall": 0.8333,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"agent_log": {
"external_commands_1pct_fpr": {
"n": 307,
"pos": 0,
"tp": 0,
"fp": 14,
"fn": 0,
"tn": 293,
"recall": null,
"fpr": 0.0456,
"fpr_wilson_upper": 0.0751,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 307,
"pos": 0,
"tp": 0,
"fp": 2,
"fn": 0,
"tn": 305,
"recall": null,
"fpr": 0.0065,
"fpr_wilson_upper": 0.0234,
"ap": null,
"auc": null
}
},
"issue_1": {
"external_commands_1pct_fpr": {
"n": 10,
"pos": 10,
"tp": 8,
"fp": 0,
"fn": 2,
"tn": 0,
"recall": 0.8,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 10,
"pos": 10,
"tp": 0,
"fp": 0,
"fn": 10,
"tn": 0,
"recall": 0.0,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"issue_2": {
"external_commands_1pct_fpr": {
"n": 10,
"pos": 10,
"tp": 10,
"fp": 0,
"fn": 0,
"tn": 0,
"recall": 1.0,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 10,
"pos": 10,
"tp": 7,
"fp": 0,
"fn": 3,
"tn": 0,
"recall": 0.7,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"issue_3": {
"external_commands_1pct_fpr": {
"n": 2,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 2,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.6576,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 2,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 2,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.6576,
"ap": null,
"auc": null
}
},
"pipe_filter": {
"external_commands_1pct_fpr": {
"n": 10,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 10,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.2775,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 10,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 10,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.2775,
"ap": null,
"auc": null
}
},
"near_miss": {
"external_commands_1pct_fpr": {
"n": 18,
"pos": 0,
"tp": 0,
"fp": 2,
"fn": 0,
"tn": 16,
"recall": null,
"fpr": 0.1111,
"fpr_wilson_upper": 0.328,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 18,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 18,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.1759,
"ap": null,
"auc": null
}
},
"routine_dev": {
"external_commands_1pct_fpr": {
"n": 13,
"pos": 0,
"tp": 0,
"fp": 4,
"fn": 0,
"tn": 9,
"recall": null,
"fpr": 0.3077,
"fpr_wilson_upper": 0.5763,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 13,
"pos": 0,
"tp": 0,
"fp": 1,
"fn": 0,
"tn": 12,
"recall": null,
"fpr": 0.0769,
"fpr_wilson_upper": 0.3331,
"ap": null,
"auc": null
}
},
"aug159_ps_download_benign": {
"external_commands_1pct_fpr": {
"n": 28,
"pos": 0,
"tp": 0,
"fp": 1,
"fn": 0,
"tn": 27,
"recall": null,
"fpr": 0.0357,
"fpr_wilson_upper": 0.1771,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 28,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 28,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.1206,
"ap": null,
"auc": null
}
},
"agent_code": {
"external_commands_1pct_fpr": {
"n": 831,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 831,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.0046,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 831,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 831,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.0046,
"ap": null,
"auc": null
}
},
"aug_heredoc_benign": {
"external_commands_1pct_fpr": {
"n": 40,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 40,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.0876,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 40,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 40,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.0876,
"ap": null,
"auc": null
}
},
"issue159_bash_exec": {
"external_commands_1pct_fpr": {
"n": 17,
"pos": 17,
"tp": 17,
"fp": 0,
"fn": 0,
"tn": 0,
"recall": 1.0,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 17,
"pos": 17,
"tp": 11,
"fp": 0,
"fn": 6,
"tn": 0,
"recall": 0.6471,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"issue159_ps_exec": {
"external_commands_1pct_fpr": {
"n": 10,
"pos": 10,
"tp": 10,
"fp": 0,
"fn": 0,
"tn": 0,
"recall": 1.0,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 10,
"pos": 10,
"tp": 7,
"fp": 0,
"fn": 3,
"tn": 0,
"recall": 0.7,
"fpr": null,
"fpr_wilson_upper": null,
"ap": null,
"auc": null
}
},
"issue159_bash_benign": {
"external_commands_1pct_fpr": {
"n": 14,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 14,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.2153,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 14,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 14,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.2153,
"ap": null,
"auc": null
}
},
"issue159_ps_benign": {
"external_commands_1pct_fpr": {
"n": 8,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 8,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.3244,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 8,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 8,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.3244,
"ap": null,
"auc": null
}
},
"heredoc_benign_fixture": {
"external_commands_1pct_fpr": {
"n": 6,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 6,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.3903,
"ap": null,
"auc": null
},
"block_0_1pct_fpr": {
"n": 6,
"pos": 0,
"tp": 0,
"fp": 0,
"fn": 0,
"tn": 6,
"recall": null,
"fpr": 0.0,
"fpr_wilson_upper": 0.3903,
"ap": null,
"auc": null
}
}
},
"issue128_combined": {
"external_commands_1pct_fpr": {
"issue_attacks_recall": "18/20",
"control_fp": 6
},
"block_0_1pct_fpr": {
"issue_attacks_recall": "7/20",
"control_fp": 1
}
}
}