eval/guardrail.json

{
 "cases": 12689,
 "families": 97,
 "level_macro_f1": 0.669,
 "level_interval": [
  0.484,
  0.781
 ],
 "level_accuracy": 0.756,
 "f1_by_level": {
  "safe": 0.989,
  "low": 0.257,
  "moderate": 0.86,
  "high": 0.508,
  "critical": 0.733
 },
 "category_macro_f1": 0.929,
 "category_interval": [
  0.724,
  0.978
 ],
 "f1_by_category": {
  "data_destruction": 0.876,
  "device_overwrite": 0.994,
  "overbroad_permissions": 0.998,
  "privilege_escalation": 1.0,
  "remote_code_execution": 1.0,
  "unsafe_download_exec": 1.0,
  "resource_exhaustion": 1.0,
  "credential_exposure": 1.0,
  "irreversible_config": 0.693,
  "history_log_tampering": 0.812,
  "network_disruption": 1.0,
  "signal_kill": 1.0,
  "shell_footgun": 0.704
 },
 "risky_f1": 0.997,
 "reversible_f1": 0.859,
 "requires_root_f1": 0.911,
 "cutoff_at_1pct_validation_fpr": 0.714,
 "test_safe_fpr": 0.0168,
 "recall_high_critical": 0.999
}