Text Classification
PEFT
Safetensors
English
safety
guard
agent-safety
agent-trajectories
lora
alert-only
Instructions to use LyzrAI/IntuitionGuard-8B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use LyzrAI/IntuitionGuard-8B with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("mistralai/Ministral-3-8B-Instruct-2512-BF16") model = PeftModel.from_pretrained(base_model, "LyzrAI/IntuitionGuard-8B") - Notebooks
- Google Colab
- Kaggle
Download evaluation/h53_ens_w25.json from LyzrAI/IntuitionGuard-8B: direct link, hf CLI and curl.
- Browser
- Download file 9.97 kB
-
https://huggingface.co/LyzrAI/IntuitionGuard-8B/resolve/main/evaluation/h53_ens_w25.json
- Command line
-
hf download hf://LyzrAI/IntuitionGuard-8B/evaluation/h53_ens_w25.json
-
curl -L -o h53_ens_w25.json https://huggingface.co/LyzrAI/IntuitionGuard-8B/resolve/main/evaluation/h53_ens_w25.json
9.97 kB
| { | |
| "train": null, | |
| "base": null, | |
| "sets": { | |
| "atbench_codex": { | |
| "n": 500, | |
| "auroc": 0.920176, | |
| "acc": 81.39999999999999, | |
| "vs_h32": { | |
| "s0": { | |
| "n": 500, | |
| "ref": { | |
| "auroc": 0.9259999999999999, | |
| "acc": 81.2 | |
| }, | |
| "acc_diff": 0.19999999999998908, | |
| "paired": { | |
| "auroc_diff": -0.00582399999999994, | |
| "auroc_ci": [ | |
| -0.017816605289016286, | |
| 0.0056432210649510085 | |
| ], | |
| "acc_diff": 0.19999999999998908, | |
| "acc_ci": [ | |
| -1.8000000000000016, | |
| 2.200000000000002 | |
| ] | |
| } | |
| }, | |
| "s1": { | |
| "n": 500, | |
| "ref": { | |
| "auroc": 0.906984, | |
| "acc": 76.8 | |
| }, | |
| "acc_diff": 4.5999999999999925, | |
| "paired": { | |
| "auroc_diff": 0.013191999999999982, | |
| "auroc_ci": [ | |
| 0.0036378436188693104, | |
| 0.02351186810063372 | |
| ], | |
| "acc_diff": 4.5999999999999925, | |
| "acc_ci": [ | |
| 2.400000000000002, | |
| 6.799999999999995 | |
| ] | |
| } | |
| }, | |
| "s2": { | |
| "n": 500, | |
| "ref": { | |
| "auroc": 0.91676, | |
| "acc": 80.4 | |
| }, | |
| "acc_diff": 0.9999999999999898, | |
| "paired": { | |
| "auroc_diff": 0.0034159999999999746, | |
| "auroc_ci": [ | |
| -0.0064309480885704285, | |
| 0.012835449918001162 | |
| ], | |
| "acc_diff": 0.9999999999999898, | |
| "acc_ci": [ | |
| -0.40000000000000036, | |
| 2.400000000000002 | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| "atbench": { | |
| "n": 1000, | |
| "auroc": 0.8341740302650896, | |
| "acc": 71.3, | |
| "vs_h32": { | |
| "s0": { | |
| "n": 1000, | |
| "ref": { | |
| "auroc": 0.8317159417739037, | |
| "acc": 73.4 | |
| }, | |
| "acc_diff": -2.100000000000002, | |
| "paired": { | |
| "auroc_diff": 0.0024580884911858947, | |
| "auroc_ci": [ | |
| -0.006278397607167122, | |
| 0.011470272398567382 | |
| ], | |
| "acc_diff": -2.100000000000002, | |
| "acc_ci": [ | |
| -4.200000000000004, | |
| 0.0 | |
| ] | |
| } | |
| }, | |
| "s1": { | |
| "n": 1000, | |
| "ref": { | |
| "auroc": 0.8275257909284734, | |
| "acc": 69.39999999999999 | |
| }, | |
| "acc_diff": 1.9000000000000017, | |
| "paired": { | |
| "auroc_diff": 0.00664823933661618, | |
| "auroc_ci": [ | |
| -0.003232769051501072, | |
| 0.01610391290726812 | |
| ], | |
| "acc_diff": 1.9000000000000017, | |
| "acc_ci": [ | |
| 0.20000000000000018, | |
| 3.600000000000003 | |
| ] | |
| } | |
| }, | |
| "s2": { | |
| "n": 1000, | |
| "ref": { | |
| "auroc": 0.8234456440431857, | |
| "acc": 72.89999999999999 | |
| }, | |
| "acc_diff": -1.6000000000000014, | |
| "paired": { | |
| "auroc_diff": 0.010728386221903974, | |
| "auroc_ci": [ | |
| -0.0003375654409279032, | |
| 0.02204627048786279 | |
| ], | |
| "acc_diff": -1.6000000000000014, | |
| "acc_ci": [ | |
| -3.6025000000000027, | |
| 0.40000000000000036 | |
| ] | |
| } | |
| } | |
| } | |
| }, | |
| "rjudge": { | |
| "n": 571, | |
| "auroc": 0.979328165374677, | |
| "acc": 92.29422066549913, | |
| "vs_h32": { | |
| "s0": { | |
| "n": 571, | |
| "ref": { | |
| "auroc": 0.9651901070505722, | |
| "acc": 87.56567425569177 | |
| }, | |
| "acc_diff": 4.728546409807355 | |
| }, | |
| "s1": { | |
| "n": 571, | |
| "ref": { | |
| "auroc": 0.9761535622000739, | |
| "acc": 91.41856392294221 | |
| }, | |
| "acc_diff": 0.8756567425569184 | |
| }, | |
| "s2": { | |
| "n": 571, | |
| "ref": { | |
| "auroc": 0.9654792666420573, | |
| "acc": 89.66725043782837 | |
| }, | |
| "acc_diff": 2.626970227670755 | |
| } | |
| } | |
| }, | |
| "assebench_strict": { | |
| "n": 1476, | |
| "auroc": 0.8805029566452119, | |
| "acc": 71.61246612466124, | |
| "vs_h32": { | |
| "s0": { | |
| "n": 1476, | |
| "ref": { | |
| "auroc": 0.8654021130905116, | |
| "acc": 65.5149051490515 | |
| }, | |
| "acc_diff": 6.097560975609751 | |
| }, | |
| "s1": { | |
| "n": 1476, | |
| "ref": { | |
| "auroc": 0.86411015722066, | |
| "acc": 68.4959349593496 | |
| }, | |
| "acc_diff": 3.1165311653116534 | |
| }, | |
| "s2": { | |
| "n": 1476, | |
| "ref": { | |
| "auroc": 0.8685138303735742, | |
| "acc": 69.10569105691057 | |
| }, | |
| "acc_diff": 2.5067750677506773 | |
| } | |
| } | |
| }, | |
| "atbench_claw_native": { | |
| "n": 500, | |
| "auroc": 0.9208233969263382, | |
| "acc": 78.8, | |
| "vs_h32": { | |
| "s0": { | |
| "n": 500, | |
| "ref": { | |
| "auroc": 0.8973320747217807, | |
| "acc": 76.6 | |
| }, | |
| "acc_diff": 2.200000000000002 | |
| }, | |
| "s1": { | |
| "n": 500, | |
| "ref": { | |
| "auroc": 0.9220488871224166, | |
| "acc": 75.2 | |
| }, | |
| "acc_diff": 3.600000000000003 | |
| }, | |
| "s2": { | |
| "n": 500, | |
| "ref": { | |
| "auroc": 0.9198877186009539, | |
| "acc": 79.80000000000001 | |
| }, | |
| "acc_diff": -1.0000000000000009 | |
| } | |
| } | |
| }, | |
| "agentdyn": { | |
| "n": 1492, | |
| "auroc": 0.957038885620481, | |
| "acc": 50.26809651474531, | |
| "vs_h32": { | |
| "s0": { | |
| "n": 1492, | |
| "ref": { | |
| "auroc": 0.975573131155941, | |
| "acc": 69.10187667560321 | |
| }, | |
| "acc_diff": -18.833780160857906 | |
| }, | |
| "s1": { | |
| "n": 1492, | |
| "ref": { | |
| "auroc": 0.971545081889435, | |
| "acc": 57.171581769437 | |
| }, | |
| "acc_diff": -6.903485254691688 | |
| }, | |
| "s2": { | |
| "n": 1492, | |
| "ref": { | |
| "auroc": 0.9655884083274818, | |
| "acc": 62.13136729222521 | |
| }, | |
| "acc_diff": -11.863270777479896 | |
| } | |
| } | |
| }, | |
| "tracesafe": { | |
| "n": 1170, | |
| "auroc": 0.5497582304526749, | |
| "acc": 56.837606837606835, | |
| "baa_0.5": 54.72222222222223, | |
| "vs_h32": { | |
| "s0": { | |
| "n": 1170, | |
| "ref": { | |
| "auroc": 0.5774228395061729, | |
| "acc": 47.008547008547005 | |
| }, | |
| "acc_diff": 9.829059829059828 | |
| }, | |
| "s1": { | |
| "n": 1170, | |
| "ref": { | |
| "auroc": 0.5978343621399177, | |
| "acc": 56.58119658119658 | |
| }, | |
| "acc_diff": 0.2564102564102555 | |
| }, | |
| "s2": { | |
| "n": 1170, | |
| "ref": { | |
| "auroc": 0.591059670781893, | |
| "acc": 47.521367521367516 | |
| }, | |
| "acc_diff": 9.316239316239317 | |
| } | |
| } | |
| } | |
| }, | |
| "latency_p50_ms": {}, | |
| "X4_calibrated_cut": { | |
| "tau": 0.68, | |
| "calibration": { | |
| "n": 699, | |
| "balanced_acc_tau": 95.85973962171457, | |
| "balanced_acc_0.5": 93.33210513387374 | |
| }, | |
| "sets": { | |
| "rjudge": { | |
| "n": 571, | |
| "acc_0.5": 92.29422066549913, | |
| "acc_tau": 89.49211908931699, | |
| "f1_0.5": 92.226148409894, | |
| "f1_tau": 89.01098901098901, | |
| "oracle_tau": 0.27, | |
| "oracle_acc": 94.04553415061297 | |
| }, | |
| "assebench_strict": { | |
| "n": 1476, | |
| "acc_0.5": 71.61246612466124, | |
| "acc_tau": 64.63414634146342, | |
| "f1_0.5": 68.66118175018698, | |
| "f1_tau": 55.61224489795919, | |
| "oracle_tau": 0.1, | |
| "oracle_acc": 80.62330623306234 | |
| }, | |
| "atbench": { | |
| "n": 1000, | |
| "acc_0.5": 71.3, | |
| "acc_tau": 76.1, | |
| "f1_0.5": 74.3978590544157, | |
| "f1_tau": 75.97989949748744, | |
| "oracle_tau": 0.74, | |
| "oracle_acc": 78.7 | |
| }, | |
| "atbench_codex": { | |
| "n": 500, | |
| "acc_0.5": 81.39999999999999, | |
| "acc_tau": 83.39999999999999, | |
| "f1_0.5": 83.99311531841653, | |
| "f1_tau": 85.1520572450805, | |
| "oracle_tau": 0.67, | |
| "oracle_acc": 83.6 | |
| } | |
| } | |
| }, | |
| "claims": { | |
| "X1_per_set": { | |
| "atbench_codex": [ | |
| -0.017816605289016286, | |
| 0.0056432210649510085 | |
| ], | |
| "atbench": [ | |
| -0.006278397607167122, | |
| 0.011470272398567382 | |
| ] | |
| }, | |
| "X1_supported": false, | |
| "X2_acc": { | |
| "rjudge": 92.29422066549913, | |
| "atbench": 71.3 | |
| }, | |
| "X2_bar": { | |
| "rjudge": 90.4, | |
| "atbench": 72.4 | |
| }, | |
| "X2_supported": false, | |
| "X3_vs_seeds_1_2": { | |
| "atbench_codex": { | |
| "s1": [ | |
| 0.0036378436188693104, | |
| 0.02351186810063372 | |
| ], | |
| "s2": [ | |
| -0.0064309480885704285, | |
| 0.012835449918001162 | |
| ] | |
| }, | |
| "atbench": { | |
| "s1": [ | |
| -0.003232769051501072, | |
| 0.01610391290726812 | |
| ], | |
| "s2": [ | |
| -0.0003375654409279032, | |
| 0.02204627048786279 | |
| ] | |
| } | |
| } | |
| }, | |
| "vs_guards": { | |
| "atbench_codex": { | |
| "agentdog_4b": { | |
| "auroc_diff": 0.08737600000000001, | |
| "auroc_ci": [ | |
| 0.05892936392245434, | |
| 0.11643739146055611 | |
| ], | |
| "acc_diff": 1.19999999999999, | |
| "acc_ci": [ | |
| -1.999999999999991, | |
| 4.399999999999993 | |
| ], | |
| "n": 500 | |
| }, | |
| "stepguard_4b": { | |
| "auroc_diff": 0.07766400000000007, | |
| "auroc_ci": [ | |
| 0.04947533703764029, | |
| 0.1067245206917969 | |
| ], | |
| "acc_diff": 3.1999999999999917, | |
| "acc_ci": [ | |
| -0.20000000000000018, | |
| 6.799999999999995 | |
| ], | |
| "n": 500 | |
| }, | |
| "agentdog15_4b": { | |
| "auroc_diff": 0.04672942783278855, | |
| "auroc_ci": [ | |
| 0.021788150792624035, | |
| 0.07281198309581959 | |
| ], | |
| "acc_diff": 3.8306451612903247, | |
| "acc_ci": [ | |
| 1.0080645161290258, | |
| 6.854838709677424 | |
| ], | |
| "n": 496 | |
| } | |
| }, | |
| "atbench": { | |
| "agentdog_4b": { | |
| "auroc_diff": 0.2316283386201905, | |
| "auroc_ci": [ | |
| 0.20211449136459286, | |
| 0.26244215874280413 | |
| ], | |
| "acc_diff": 11.099999999999998, | |
| "acc_ci": [ | |
| 8.200000000000006, | |
| 13.900000000000002 | |
| ], | |
| "n": 1000 | |
| }, | |
| "stepguard_4b": { | |
| "auroc_diff": 0.04670168126052543, | |
| "auroc_ci": [ | |
| 0.018970855969553564, | |
| 0.07253321422256387 | |
| ], | |
| "acc_diff": -1.7000000000000015, | |
| "acc_ci": [ | |
| -4.902500000000003, | |
| 1.6000000000000014 | |
| ], | |
| "n": 1000 | |
| }, | |
| "agentdog15_4b": { | |
| "auroc_diff": 0.0910115919724459, | |
| "auroc_ci": [ | |
| 0.06327126043629389, | |
| 0.11848111068178413 | |
| ], | |
| "acc_diff": 3.549695740365111, | |
| "acc_ci": [ | |
| 0.40567951318457585, | |
| 6.592292089249496 | |
| ], | |
| "n": 986 | |
| } | |
| }, | |
| "rjudge": { | |
| "agentdog_4b": { | |
| "auroc_diff": 0.012193921496247206, | |
| "auroc_ci": [ | |
| -0.0013734734750716765, | |
| 0.027566819235545093 | |
| ], | |
| "acc_diff": 3.8528896672504365, | |
| "acc_ci": [ | |
| 0.5210157618213603, | |
| 7.005253940455347 | |
| ], | |
| "n": 571 | |
| }, | |
| "stepguard_4b": { | |
| "auroc_diff": 0.04687461547926697, | |
| "auroc_ci": [ | |
| 0.029323098673131785, | |
| 0.0656522129420753 | |
| ], | |
| "acc_diff": 2.8021015761821366, | |
| "acc_ci": [ | |
| 0.5253940455341444, | |
| 5.078809106830128 | |
| ], | |
| "n": 571 | |
| }, | |
| "agentdog15_4b": { | |
| "auroc_diff": 0.009093146302448618, | |
| "auroc_ci": [ | |
| 0.0010304723577039265, | |
| 0.017428346840814703 | |
| ], | |
| "acc_diff": 1.9264448336252182, | |
| "acc_ci": [ | |
| -0.5253940455341444, | |
| 4.20315236427321 | |
| ], | |
| "n": 571 | |
| } | |
| } | |
| } | |
| } |