Text Classification
PEFT
Safetensors
English
safety
guard
agent-safety
agent-trajectories
lora
alert-only
fast
Instructions to use LyzrAI/IntuitionGuard-3B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use LyzrAI/IntuitionGuard-3B with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("mistralai/Ministral-3-3B-Instruct-2512-BF16") model = PeftModel.from_pretrained(base_model, "LyzrAI/IntuitionGuard-3B") - Notebooks
- Google Colab
- Kaggle
IntuitionGuard-3B: three w25 seed adapters (H54), model card, scorer, evaluation reports
ab3b173 verified Download evaluation/h49.json from LyzrAI/IntuitionGuard-3B: direct link, hf CLI and curl.
- Browser
- Download file 55.3 kB
-
https://huggingface.co/LyzrAI/IntuitionGuard-3B/resolve/main/evaluation/h49.json
- Command line
-
hf download hf://LyzrAI/IntuitionGuard-3B/evaluation/h49.json
-
curl -L -o h49.json https://huggingface.co/LyzrAI/IntuitionGuard-3B/resolve/main/evaluation/h49.json
55.3 kB
{
"sets": {
"rjudge": {
"n": 571,
"missing": 0,
"auroc": 0.9573151224314015,
"acc": 90.01751313485113,
"f1": 90.2229845626072,
"published": 90.4,
"reproduced": true
},
"assebench_strict": {
"n": 1476,
"missing": 0,
"auroc": 0.8310004016927144,
"acc": 79.53929539295393,
"f1": 79.45578231292517
},
"atbench": {
"n": 1000,
"missing": 0,
"auroc": 0.8438823797656717,
"acc": 78.3,
"f1": 77.41935483870968,
"published": 78.4,
"reproduced": true
},
"atbench_codex": {
"n": 500,
"missing": 0,
"auroc": 0.893264,
"acc": 83.2,
"f1": 84.21052631578947
},
"atbench_claw": {
"n": 500,
"missing": 0,
"auroc": 0.8878759273979862,
"acc": 87.6,
"f1": 89.8360655737705
},
"atbench_claw_native": {
"n": 500,
"missing": 0,
"auroc": 0.8906581213566508,
"acc": 88.8,
"f1": 90.81967213114754
},
"agentdyn": {
"n": 1486,
"missing": 4864,
"auroc": 0.6792305885848754,
"acc": 44.751009421265145,
"f1": 27.409372236958447
},
"tracesafe": {
"n": 1170,
"missing": 0,
"auroc": 0.5799948559670782,
"acc": 61.965811965811966,
"f1": 75.50908090258667,
"baa_0.5": 53.425925925925924
}
},
"vs_guards": {
"rjudge": {
"agentdog_4b": {
"auroc_diff": -0.009819121447028345,
"auroc_ci": [
-0.029417630933077036,
0.01036861132294604
],
"acc_diff": 1.5761821366024442,
"acc_ci": [
-1.5761821366024553,
4.728546409807355
],
"n": 571
},
"stepguard_4b": {
"auroc_diff": 0.024861572535991416,
"auroc_ci": [
0.0038273730049030066,
0.046898997689975214
],
"acc_diff": 0.5253940455341444,
"acc_ci": [
-1.7513134851138368,
2.977232924693518
],
"n": 571
},
"agentdog15_4b": {
"auroc_diff": -0.012919896640826933,
"auroc_ci": [
-0.027970378301906497,
0.001484768215849368
],
"acc_diff": -0.350262697022774,
"acc_ci": [
-2.806479859894921,
2.1015761821365997
],
"n": 571
}
},
"atbench": {
"agentdog_4b": {
"auroc_diff": 0.24133668812077258,
"auroc_ci": [
0.20753934766084922,
0.27539971488993115
],
"acc_diff": 18.100000000000005,
"acc_ci": [
14.497500000000002,
21.70000000000001
],
"n": 1000
},
"stepguard_4b": {
"auroc_diff": 0.05641003076110751,
"auroc_ci": [
0.027528281897365746,
0.08525293445577453
],
"acc_diff": 5.300000000000004,
"acc_ci": [
2.6000000000000023,
8.200000000000006
],
"n": 1000
},
"agentdog15_4b": {
"auroc_diff": 0.10157892787628642,
"auroc_ci": [
0.07303585823634727,
0.1287921679333249
],
"acc_diff": 10.649087221095332,
"acc_ci": [
7.505070993914808,
13.691683569979718
],
"n": 986
}
},
"atbench_codex": {
"agentdog_4b": {
"auroc_diff": 0.06046399999999996,
"auroc_ci": [
0.026949525969834954,
0.09593771105659839
],
"acc_diff": 2.9999999999999916,
"acc_ci": [
-0.40000000000000036,
6.599999999999994
],
"n": 500
},
"stepguard_4b": {
"auroc_diff": 0.05075200000000002,
"auroc_ci": [
0.0193557043528508,
0.08398019999999978
],
"acc_diff": 4.999999999999993,
"acc_ci": [
1.5999999999999903,
8.599999999999996
],
"n": 500
},
"agentdog15_4b": {
"auroc_diff": 0.018779571728208322,
"auroc_ci": [
-0.012691299924244883,
0.05113527583531361
],
"acc_diff": 5.645161290322587,
"acc_ci": [
2.016129032258074,
9.2741935483871
],
"n": 496
}
}
},
"arms": {
"h32_s0": {
"rjudge": {
"auroc_diff": 0.007874984619170688,
"auroc_ci": [
-0.007952686688214841,
0.024295175703981976
],
"acc_diff": -2.4518388791593626,
"acc_ci": [
-4.903677758318747,
0.0
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.03440171139779724,
"auroc_ci": [
0.014536256936952666,
0.05408362907439627
],
"acc_diff": -14.02439024390244,
"acc_ci": [
-16.327913279132787,
-11.788617886178866
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.012166437991767975,
"auroc_ci": [
-0.03570004917010014,
0.01121509092667737
],
"acc_diff": -4.900000000000004,
"acc_ci": [
-7.700000000000006,
-2.100000000000002
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.03273599999999999,
"auroc_ci": [
0.007647562328205493,
0.05731726698312903
],
"acc_diff": -1.9999999999999907,
"acc_ci": [
-4.999999999999993,
1.19999999999999
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.0884009009009008,
"auroc_ci": [
-0.1353539010617551,
-0.04307528012113371
],
"acc_diff": -17.400000000000006,
"acc_ci": [
-21.799999999999997,
-13.200000000000001
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.006673953365129881,
"auroc_ci": [
-0.016476271766343024,
0.0302829514170535
],
"acc_diff": -12.2,
"acc_ci": [
-15.800000000000002,
-8.600000000000009
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.3023584611771697,
"auroc_ci": [
0.2713873845425281,
0.3321378377437629
],
"acc_diff": 24.29340511440108,
"acc_ci": [
22.139973082099594,
26.446837146702556
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.002572016460905324,
"auroc_ci": [
-0.05531056109153116,
0.05193769749625696
],
"acc_diff": -14.95726495726496,
"acc_ci": [
-17.863247863247867,
-11.965811965811973
],
"n": 1170
}
},
"h32_s1": {
"rjudge": {
"auroc_diff": 0.018838439768672388,
"auroc_ci": [
0.003325026556980132,
0.03512521575090488
],
"acc_diff": 1.4010507880910739,
"acc_ci": [
-0.8800350262697026,
3.8528896672504365
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.033109755527945595,
"auroc_ci": [
0.012206340680426175,
0.05325250068786603
],
"acc_diff": -11.043360433604343,
"acc_ci": [
-13.34688346883468,
-8.738143631436314
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.01635658883719826,
"auroc_ci": [
-0.040664058974714444,
0.006672728700192823
],
"acc_diff": -8.900000000000007,
"acc_ci": [
-11.899999999999999,
-5.900000000000006
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.013720000000000065,
"auroc_ci": [
-0.014216963202614254,
0.03959793810315116
],
"acc_diff": -6.399999999999995,
"acc_ci": [
-9.999999999999998,
-2.9999999999999916
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.047810678325384104,
"auroc_ci": [
-0.09032133512865875,
-0.005748119866717939
],
"acc_diff": -16.200000000000003,
"acc_ci": [
-20.599999999999994,
-12.4
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.03139076576576583,
"auroc_ci": [
0.009191080144844165,
0.055169730282897005
],
"acc_diff": -13.600000000000001,
"acc_ci": [
-17.400000000000006,
-9.999999999999998
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.29804507962981597,
"auroc_ci": [
0.26761653461665763,
0.32717298082038915
],
"acc_diff": 12.314939434724092,
"acc_ci": [
10.296096904441454,
14.333781965006725
],
"n": 1486
},
"tracesafe": {
"auroc_diff": 0.01783950617283947,
"auroc_ci": [
-0.04135183514696254,
0.07797434311313615
],
"acc_diff": -5.384615384615388,
"acc_ci": [
-7.948717948717943,
-2.564102564102566
],
"n": 1170
}
},
"h32_s2": {
"rjudge": {
"auroc_diff": 0.00816414421065581,
"auroc_ci": [
-0.006731936641794512,
0.024024713236699097
],
"acc_diff": -0.3502626970227629,
"acc_ci": [
-2.8021015761821366,
1.9264448336252185
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.03751342868085983,
"auroc_ci": [
0.016719410285598747,
0.058187898194103735
],
"acc_diff": -10.433604336043366,
"acc_ci": [
-12.601626016260159,
-8.265582655826552
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.020436735722486055,
"auroc_ci": [
-0.045602033180343796,
0.003840008888065487
],
"acc_diff": -5.400000000000005,
"acc_ci": [
-8.500000000000007,
-2.500000000000002
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.023496000000000072,
"auroc_ci": [
-0.0017409265232811554,
0.04835289481209381
],
"acc_diff": -2.7999999999999914,
"acc_ci": [
-5.999999999999995,
0.40000000000000036
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.07381094329623727,
"auroc_ci": [
-0.11887612070147412,
-0.029410936924792776
],
"acc_diff": -14.000000000000002,
"acc_ci": [
-18.200000000000006,
-9.999999999999998
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.029229597244303074,
"auroc_ci": [
0.005686636857774261,
0.05460499125812438
],
"acc_diff": -8.999999999999996,
"acc_ci": [
-12.4,
-5.999999999999995
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.2921285579406957,
"auroc_ci": [
0.2619928308751839,
0.32060527121123084
],
"acc_diff": 17.29475100942126,
"acc_ci": [
15.208613728129205,
19.17900403768506
],
"n": 1486
},
"tracesafe": {
"auroc_diff": 0.011064814814814805,
"auroc_ci": [
-0.04049923614432227,
0.06602622144004643
],
"acc_diff": -14.444444444444448,
"acc_ci": [
-17.179487179487186,
-11.623931623931627
],
"n": 1170
}
},
"teach_s0": {
"rjudge": {
"auroc_diff": 0.026664205733973212,
"auroc_ci": [
0.011744003450816496,
0.042855352720441636
],
"acc_diff": 3.1523642732049106,
"acc_ci": [
1.0507880910682998,
5.604203152364273
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.038058983437181726,
"auroc_ci": [
0.0180024283043435,
0.05757223622524813
],
"acc_diff": -9.55284552845529,
"acc_ci": [
-11.720867208672093,
-7.3848238482384865
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.04474161069798532,
"auroc_ci": [
-0.0712638574041776,
-0.01988803078884324
],
"acc_diff": -7.300000000000006,
"acc_ci": [
-10.20000000000001,
-4.397500000000018
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.017967999999999984,
"auroc_ci": [
-0.009788141231142172,
0.043838586328408806
],
"acc_diff": -0.6000000000000005,
"acc_ci": [
-3.599999999999992,
2.599999999999991
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.05999105723370435,
"auroc_ci": [
-0.1037707087780301,
-0.016298044578733725
],
"acc_diff": -12.8,
"acc_ci": [
-16.800000000000004,
-8.999999999999996
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.030388844727079922,
"auroc_ci": [
0.007636752082114467,
0.0544938358970897
],
"acc_diff": -7.599999999999996,
"acc_ci": [
-10.599999999999998,
-4.600000000000004
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.21798871811093057,
"auroc_ci": [
0.18676343194155612,
0.24792080162965838
],
"acc_diff": 0.538358008075368,
"acc_ci": [
-1.211305518169581,
2.2880215343203227
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.05831275720164619,
"auroc_ci": [
-0.11687461041779995,
-0.0007434666158338158
],
"acc_diff": -8.461538461538465,
"acc_ci": [
-10.940170940170946,
-5.897435897435899
],
"n": 1170
}
},
"teach_s1": {
"rjudge": {
"auroc_diff": 0.024110988064476424,
"auroc_ci": [
0.00915685507969467,
0.04010427955372052
],
"acc_diff": 3.5026269702276736,
"acc_ci": [
1.0507880910682998,
5.954465849387036
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.03995441254775933,
"auroc_ci": [
0.019684339264772234,
0.06042728154015862
],
"acc_diff": -4.40379403794039,
"acc_ci": [
-6.57350948509485,
-2.2357723577235755
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.03151913468884893,
"auroc_ci": [
-0.055698101229814195,
-0.008391155960339345
],
"acc_diff": -11.299999999999999,
"acc_ci": [
-14.400000000000002,
-8.000000000000007
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.013072000000000084,
"auroc_ci": [
-0.013858453292753726,
0.03908661769486731
],
"acc_diff": -3.7999999999999923,
"acc_ci": [
-7.199999999999996,
-0.40000000000000036
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.06133247217806015,
"auroc_ci": [
-0.10408844679606112,
-0.016616533974221102
],
"acc_diff": -13.4,
"acc_ci": [
-17.400000000000006,
-9.595000000000026
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.027258876523582276,
"auroc_ci": [
0.004533656107525028,
0.05203776677225913
],
"acc_diff": -9.199999999999998,
"acc_ci": [
-12.4,
-6.000000000000005
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.2809736739946226,
"auroc_ci": [
0.2516919730235246,
0.309084246694381
],
"acc_diff": -4.441453566621806,
"acc_ci": [
-6.1911170928667545,
-2.759084791386268
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.0335185185185185,
"auroc_ci": [
-0.09381174802031764,
0.027280285613804252
],
"acc_diff": 2.3931623931623958,
"acc_ci": [
0.0,
4.700854700854706
],
"n": 1170
}
},
"teach_s2": {
"rjudge": {
"auroc_diff": 0.02253599114064231,
"auroc_ci": [
0.008199521078188624,
0.03761176078154388
],
"acc_diff": 2.101576182136611,
"acc_ci": [
-0.17513134851138146,
4.378283712784581
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.0416022868459649,
"auroc_ci": [
0.022631654817631023,
0.06038439355360588
],
"acc_diff": -6.165311653116534,
"acc_ci": [
-8.19783197831978,
-4.132791327913276
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.04098747554911997,
"auroc_ci": [
-0.0681189814160153,
-0.017208530982927295
],
"acc_diff": -8.000000000000007,
"acc_ci": [
-11.099999999999998,
-5.197500000000018
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.032056000000000084,
"auroc_ci": [
0.005959563984709096,
0.056824524737431235
],
"acc_diff": 0.20000000000000018,
"acc_ci": [
-2.8000000000000025,
3.400000000000003
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.0756243375728669,
"auroc_ci": [
-0.12093549248642425,
-0.030848845982364478
],
"acc_diff": -15.000000000000002,
"acc_ci": [
-19.399999999999995,
-10.799999999999999
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.015244104398515979,
"auroc_ci": [
-0.0072457614942529,
0.03926520807191212
],
"acc_diff": -11.799999999999999,
"acc_ci": [
-15.600000000000003,
-8.399999999999997
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.25762869453111137,
"auroc_ci": [
0.22860609110139235,
0.28681301151543037
],
"acc_diff": -4.845222072678334,
"acc_ci": [
-6.393001345895022,
-3.2974427994616473
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.021430041152263435,
"auroc_ci": [
-0.08340559165120655,
0.040954313259502
],
"acc_diff": 2.478632478632481,
"acc_ci": [
0.2564102564102555,
4.702991452991447
],
"n": 1170
}
},
"w25": {
"rjudge": {
"auroc_diff": 0.027297895902547054,
"auroc_ci": [
0.012621611780857056,
0.043657088143969285
],
"acc_diff": 0.8756567425569184,
"acc_ci": [
-1.2259194395796924,
3.1523642732048995
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.04647491288873118,
"auroc_ci": [
0.026748617806584617,
0.06603242654899324
],
"acc_diff": -13.482384823848248,
"acc_ci": [
-15.650406504065039,
-11.314363143631434
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.021236764523523077,
"auroc_ci": [
-0.045535106987388815,
0.0012045599563388937
],
"acc_diff": -6.400000000000006,
"acc_ci": [
-9.200000000000008,
-3.400000000000003
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.024720000000000075,
"auroc_ci": [
-0.00038560806857672325,
0.04928488078081273
],
"acc_diff": -0.40000000000000036,
"acc_ci": [
-3.399999999999992,
2.7999999999999914
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.04604696608373082,
"auroc_ci": [
-0.08747025206226479,
-0.004249828857385338
],
"acc_diff": -11.6,
"acc_ci": [
-15.600000000000003,
-7.9999999999999964
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.023930180180180227,
"auroc_ci": [
0.0010412177613762315,
0.047593642037737006
],
"acc_diff": -7.599999999999996,
"acc_ci": [
-10.600000000000009,
-4.600000000000004
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.28149367610338694,
"auroc_ci": [
0.25211135864905704,
0.3106064104066566
],
"acc_diff": 10.026917900403765,
"acc_ci": [
8.07537012113056,
11.845558546433368
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.05531893004115229,
"auroc_ci": [
-0.11433787270642207,
0.002988992890915766
],
"acc_diff": -15.64102564102564,
"acc_ci": [
-18.46367521367522,
-12.820512820512825
],
"n": 1170
}
},
"w25_s1": {
"rjudge": {
"auroc_diff": 0.021428571428571463,
"auroc_ci": [
0.007203129469589909,
0.03743749909351522
],
"acc_diff": 1.5761821366024553,
"acc_ci": [
-0.8756567425569184,
4.028021015761829
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.040473810569189306,
"auroc_ci": [
0.01948081899076893,
0.06048275263461059
],
"acc_diff": -6.77506775067751,
"acc_ci": [
-9.010840108401085,
-4.607046070460696
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.020466736802524976,
"auroc_ci": [
-0.04476283331154423,
0.002082946088219829
],
"acc_diff": -10.399999999999999,
"acc_ci": [
-13.5025,
-7.199999999999996
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.011152000000000162,
"auroc_ci": [
-0.018649670254575257,
0.038970926940538976
],
"acc_diff": -5.199999999999994,
"acc_ci": [
-8.799999999999997,
-1.7999999999999905
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.05551139374668779,
"auroc_ci": [
-0.09812222191422454,
-0.012014932463994986
],
"acc_diff": -13.8,
"acc_ci": [
-17.800000000000004,
-9.799999999999997
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.037037957074721684,
"auroc_ci": [
0.015310338720601623,
0.06137738375130607
],
"acc_diff": -9.999999999999998,
"acc_ci": [
-13.200000000000001,
-6.799999999999995
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.2797227933458901,
"auroc_ci": [
0.24973473654440007,
0.3081108558164418
],
"acc_diff": 0.8748317631224745,
"acc_ci": [
-0.8748317631224745,
2.4899057873485897
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.017227366255144094,
"auroc_ci": [
-0.07700874705521323,
0.04091141798973637
],
"acc_diff": -2.7350427350427364,
"acc_ci": [
-5.301282051282055,
-0.2564102564102555
],
"n": 1170
}
},
"w25_s2": {
"rjudge": {
"auroc_diff": 0.015183954718838488,
"auroc_ci": [
0.00047657321685243287,
0.03083053817962469
],
"acc_diff": 2.8021015761821366,
"acc_ci": [
0.5253940455341444,
5.078809106830128
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.05083000924827408,
"auroc_ci": [
0.03150054163328806,
0.06927791401082924
],
"acc_diff": -4.20054200542006,
"acc_ci": [
-6.165311653116534,
-2.2357723577235755
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.008218295858651081,
"auroc_ci": [
-0.03286021643952745,
0.01577692383379831
],
"acc_diff": -5.300000000000004,
"acc_ci": [
-8.300000000000008,
-2.500000000000002
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.018208000000000002,
"auroc_ci": [
-0.006853488877162081,
0.04265247811314437
],
"acc_diff": -2.199999999999991,
"acc_ci": [
-5.199999999999994,
1.0000000000000009
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.07412559618441972,
"auroc_ci": [
-0.12038361289401471,
-0.028799876174581795
],
"acc_diff": -17.800000000000004,
"acc_ci": [
-22.199999999999996,
-13.600000000000001
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.024070945945945832,
"auroc_ci": [
0.0010107289280552939,
0.04834301867031254
],
"acc_diff": -12.2,
"acc_ci": [
-15.805000000000001,
-8.799999999999997
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.2758191830455349,
"auroc_ci": [
0.24683172399746614,
0.3043445292061223
],
"acc_diff": 4.777927321668907,
"acc_ci": [
2.960969044414541,
6.594885598923284
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.014346707818930104,
"auroc_ci": [
-0.07047452223857836,
0.04304651186572272
],
"acc_diff": 2.991452991452992,
"acc_ci": [
0.5982905982905962,
5.555555555555558
],
"n": 1170
}
},
"w25u": {
"rjudge": {
"auroc_diff": 0.008994708994709,
"auroc_ci": [
-0.0067606212609408415,
0.02496392057762967
],
"acc_diff": 1.0507880910682998,
"acc_ci": [
-1.0507880910682998,
3.1523642732049106
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.04319878184348935,
"auroc_ci": [
0.02373424861880531,
0.06272749432351682
],
"acc_diff": -9.688346883468835,
"acc_ci": [
-11.788617886178853,
-7.72357723577236
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.012290442455928585,
"auroc_ci": [
-0.03481184095060911,
0.008839146420084905
],
"acc_diff": -4.600000000000004,
"acc_ci": [
-7.400000000000007,
-1.9000000000000017
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.02254400000000012,
"auroc_ci": [
-0.001617301510464844,
0.046240166942443615
],
"acc_diff": -1.0000000000000009,
"acc_ci": [
-3.7999999999999923,
1.999999999999991
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.033601616322204486,
"auroc_ci": [
-0.07546856229137383,
0.007421089313318868
],
"acc_diff": -13.8,
"acc_ci": [
-17.800000000000004,
-9.999999999999998
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.04160042395336505,
"auroc_ci": [
0.01904934817777225,
0.06656498215558034
],
"acc_diff": -10.199999999999998,
"acc_ci": [
-13.405,
-6.999999999999995
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.26762136178248097,
"auroc_ci": [
0.2382663703585987,
0.29672721837869803
],
"acc_diff": 2.893674293405113,
"acc_ci": [
1.3458950201884257,
4.441453566621806
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.00508230452674896,
"auroc_ci": [
-0.0590298745773972,
0.04828723445989927
],
"acc_diff": -4.7863247863247915,
"acc_ci": [
-7.179487179487188,
-2.3076923076923106
],
"n": 1170
}
},
"w25_18k": {
"rjudge": {
"auroc_diff": 0.01794635166728198,
"auroc_ci": [
0.00312814569761907,
0.033386208316901166
],
"acc_diff": 1.5761821366024553,
"acc_ci": [
-0.7005253940455369,
3.857267950963208
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.03465206871747917,
"auroc_ci": [
0.014353688748294865,
0.05431376352931247
],
"acc_diff": -6.368563685636863,
"acc_ci": [
-8.536585365853655,
-4.268292682926833
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.010584381037717527,
"auroc_ci": [
-0.0339783419882701,
0.012431406432390258
],
"acc_diff": -3.7000000000000033,
"acc_ci": [
-6.400000000000006,
-1.0000000000000009
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.032784000000000035,
"auroc_ci": [
0.007951096236559086,
0.05746341749322339
],
"acc_diff": -2.7999999999999914,
"acc_ci": [
-5.999999999999995,
0.40000000000000036
],
"n": 500
},
"atbench_claw": {
"auroc_diff": 0.029262718600953974,
"auroc_ci": [
-0.0013578393814041437,
0.06156798416228268
],
"acc_diff": -5.000000000000004,
"acc_ci": [
-8.399999999999997,
-1.8000000000000016
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.04006856120826707,
"auroc_ci": [
0.018364333178678822,
0.06341901469382968
],
"acc_diff": -4.200000000000004,
"acc_ci": [
-6.800000000000006,
-1.6000000000000014
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.3126147239675441,
"auroc_ci": [
0.28160570135852625,
0.34312658732202345
],
"acc_diff": 11.37281292059219,
"acc_ci": [
9.42126514131898,
13.25706594885599
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.019089506172839554,
"auroc_ci": [
-0.08283397000943271,
0.0450631162242672
],
"acc_diff": -6.41025641025641,
"acc_ci": [
-8.803418803418806,
-3.8461538461538436
],
"n": 1170
}
},
"gold": {
"rjudge": {
"auroc_diff": -0.037289282638119925,
"auroc_ci": [
-0.05610189636446871,
-0.020278410559548307
],
"acc_diff": -5.2539404553415,
"acc_ci": [
-8.056042031523647,
-2.626970227670755
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.01216755257036628,
"auroc_ci": [
-0.011895602495230992,
0.03578209939713676
],
"acc_diff": -15.108401084010847,
"acc_ci": [
-17.479674796747968,
-12.60162601626017
],
"n": 1476
},
"atbench": {
"auroc_diff": 0.00901832465968766,
"auroc_ci": [
-0.018136955554487723,
0.03545533682789022
],
"acc_diff": -5.800000000000005,
"acc_ci": [
-9.100000000000009,
-2.497500000000016
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": -0.008975999999999984,
"auroc_ci": [
-0.03999844521269604,
0.02135457202701558
],
"acc_diff": -15.199999999999992,
"acc_ci": [
-19.599999999999994,
-10.999999999999998
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.05898913619501844,
"auroc_ci": [
-0.10452264917653872,
-0.01267085371400443
],
"acc_diff": -20.799999999999997,
"acc_ci": [
-25.4,
-16.39999999999999
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.03184618441971376,
"auroc_ci": [
0.005354780299134074,
0.05881589469400627
],
"acc_diff": -15.600000000000003,
"acc_ci": [
-19.600000000000005,
-11.799999999999999
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.3043785615351805,
"auroc_ci": [
0.27341592250735086,
0.33398096554827644
],
"acc_diff": 44.34724091520861,
"acc_ci": [
41.790040376850605,
46.90444145356661
],
"n": 1486
},
"tracesafe": {
"auroc_diff": 0.05336419753086419,
"auroc_ci": [
-0.0065034560355295866,
0.11128625064757817
],
"acc_diff": -29.572649572649574,
"acc_ci": [
-32.9059829059829,
-26.239316239316242
],
"n": 1170
}
},
"3b_mix": {
"rjudge": {
"auroc_diff": 0.009130060292851017,
"auroc_ci": [
-0.005417968121949132,
0.024178117695591493
],
"acc_diff": -1.0507880910682998,
"acc_ci": [
-3.502626970227663,
1.4010507880910739
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.03561706540117893,
"auroc_ci": [
0.014599626301613725,
0.05611238535852722
],
"acc_diff": -15.311653116531165,
"acc_ci": [
-17.75067750677507,
-12.87262872628726
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.024686888727994405,
"auroc_ci": [
-0.049083078753780846,
0.00031826997567228205
],
"acc_diff": -8.300000000000008,
"acc_ci": [
-11.299999999999999,
-5.397500000000019
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": -0.007399999999999962,
"auroc_ci": [
-0.03835454567452978,
0.02069721745066948
],
"acc_diff": -4.999999999999993,
"acc_ci": [
-8.199999999999996,
-1.6000000000000012
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.040333532061473165,
"auroc_ci": [
-0.08366551061764192,
0.0015882133881504662
],
"acc_diff": -17.600000000000005,
"acc_ci": [
-21.999999999999996,
-13.200000000000001
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.034487612612612684,
"auroc_ci": [
0.010959511729462357,
0.05876145691266628
],
"acc_diff": -12.6,
"acc_ci": [
-16.400000000000002,
-9.199999999999998
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.30546888853742815,
"auroc_ci": [
0.27421938528026124,
0.3357120191047023
],
"acc_diff": 13.324360699865412,
"acc_ci": [
11.50740242261104,
15.210296096904438
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.04898662551440336,
"auroc_ci": [
-0.10335903707603565,
0.005058694408387157
],
"acc_diff": -14.188034188034187,
"acc_ci": [
-17.00854700854701,
-11.450854700854705
],
"n": 1170
}
},
"3b_teach": {
"rjudge": {
"auroc_diff": 0.016992740248554128,
"auroc_ci": [
0.0025182511123934105,
0.032441137542234814
],
"acc_diff": 1.4010507880910739,
"acc_ci": [
-0.7005253940455369,
3.507005253940434
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.040397208702719456,
"auroc_ci": [
0.0203002812293044,
0.05971556009090536
],
"acc_diff": -11.44986449864499,
"acc_ci": [
-13.617886178861793,
-9.281842818428188
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.05587801160841799,
"auroc_ci": [
-0.08104927705274337,
-0.03068295305432122
],
"acc_diff": -10.799999999999999,
"acc_ci": [
-13.900000000000002,
-7.800000000000007
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": -0.005856000000000083,
"auroc_ci": [
-0.033976692102995795,
0.022046371009919456
],
"acc_diff": -4.799999999999994,
"acc_ci": [
-8.199999999999996,
-1.4000000000000012
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.03600291467938521,
"auroc_ci": [
-0.07600106832714432,
0.004992177833657996
],
"acc_diff": -11.399999999999999,
"acc_ci": [
-15.600000000000003,
-7.599999999999996
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.016925013248542675,
"auroc_ci": [
-0.007029184936195254,
0.042088964622115047
],
"acc_diff": -5.400000000000005,
"acc_ci": [
-8.205000000000005,
-2.8000000000000025
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.27634877044662665,
"auroc_ci": [
0.24634185587341254,
0.3064234033517196
],
"acc_diff": -1.8842530282637937,
"acc_ci": [
-3.6339165545087484,
-0.0672947510094224
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.0523508230452675,
"auroc_ci": [
-0.11615654413509452,
0.00825182301659372
],
"acc_diff": -6.75213675213675,
"acc_ci": [
-9.487179487179487,
-3.9316239316239288
],
"n": 1170
}
},
"ens_w25": {
"rjudge": {
"auroc_diff": 0.02201304294327555,
"auroc_ci": [
0.007454665057679738,
0.03791683641457943
],
"acc_diff": 2.2767075306479923,
"acc_ci": [
0.0,
4.728546409807355
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.04950255495249756,
"auroc_ci": [
0.030079140399036464,
0.06825665201986864
],
"acc_diff": -7.92682926829269,
"acc_ci": [
-10.162601626016254,
-5.7588075880758876
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.00970834950058208,
"auroc_ci": [
-0.03346510544943757,
0.013041593711894398
],
"acc_diff": -7.000000000000006,
"acc_ci": [
-9.90000000000001,
-4.200000000000004
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.026912000000000047,
"auroc_ci": [
0.0015388753759192797,
0.051465866128847744
],
"acc_diff": -1.8000000000000016,
"acc_ci": [
-4.799999999999994,
1.200000000000001
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.05609929782723877,
"auroc_ci": [
-0.09980937130359255,
-0.013004912028252665
],
"acc_diff": -13.4,
"acc_ci": [
-17.400000000000006,
-9.399999999999997
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.0301652755696874,
"auroc_ci": [
0.00826929534522002,
0.053885153659750266
],
"acc_diff": -9.999999999999998,
"acc_ci": [
-13.4,
-6.799999999999995
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.28336760075340406,
"auroc_ci": [
0.25365637678009606,
0.3123266857844479
],
"acc_diff": 5.383580080753703,
"acc_ci": [
3.7668236877523524,
7.133243606998652
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.030236625514403315,
"auroc_ci": [
-0.08858884469752795,
0.026978939993188088
],
"acc_diff": -5.128205128205132,
"acc_ci": [
-7.606837606837602,
-2.564102564102555
],
"n": 1170
}
},
"ens5": {
"rjudge": {
"auroc_diff": 0.020967146548541926,
"auroc_ci": [
0.006367251053043199,
0.03658942749894822
],
"acc_diff": 1.5761821366024553,
"acc_ci": [
-0.5253940455341555,
3.677758318739055
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.048985025269274285,
"auroc_ci": [
0.02968558198250373,
0.06764499155631172
],
"acc_diff": -8.265582655826565,
"acc_ci": [
-10.367547425474262,
-6.165311653116534
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.004636166902008632,
"auroc_ci": [
-0.027443739316762052,
0.017501228078596984
],
"acc_diff": -5.800000000000005,
"acc_ci": [
-8.600000000000009,
-3.0000000000000027
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.030208000000000013,
"auroc_ci": [
0.006398239792704952,
0.05452742921964889
],
"acc_diff": -1.6000000000000014,
"acc_ci": [
-4.5999999999999925,
1.4000000000000012
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.028078630100688784,
"auroc_ci": [
-0.06718745088943334,
0.011142848057641068
],
"acc_diff": -13.0,
"acc_ci": [
-17.000000000000004,
-9.199999999999998
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.03717044250132484,
"auroc_ci": [
0.015742992350449105,
0.06097563296146095
],
"acc_diff": -8.599999999999996,
"acc_ci": [
-11.799999999999999,
-5.595000000000022
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.30230813839245063,
"auroc_ci": [
0.2718789639962889,
0.3317150874352668
],
"acc_diff": 6.3257065948855935,
"acc_ci": [
4.7089502018842495,
8.008075370121132
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.020787037037037104,
"auroc_ci": [
-0.07810256011557502,
0.03735814266191144
],
"acc_diff": -4.2735042735042805,
"acc_ci": [
-6.75213675213675,
-1.7948717948717996
],
"n": 1170
}
},
"3b_w25": {
"rjudge": {
"auroc_diff": -0.0001538082933432161,
"auroc_ci": [
-0.015089005392104194,
0.014787800188764985
],
"acc_diff": -1.7513134851138368,
"acc_ci": [
-4.028021015761829,
0.5253940455341555
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.03768624996496861,
"auroc_ci": [
0.01664732173408001,
0.05790802715795375
],
"acc_diff": -13.27913279132792,
"acc_ci": [
-15.582655826558266,
-11.041666666666673
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.03211115600161629,
"auroc_ci": [
-0.057444702762109896,
-0.00788665354918257
],
"acc_diff": -9.300000000000008,
"acc_ci": [
-12.4,
-6.400000000000006
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.016959999999999975,
"auroc_ci": [
-0.010081487959799804,
0.04343733246669551
],
"acc_diff": -2.9999999999999916,
"acc_ci": [
-5.999999999999995,
0.20000000000000018
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.05013745363010069,
"auroc_ci": [
-0.09312810911831905,
-0.00886626948528849
],
"acc_diff": -14.400000000000002,
"acc_ci": [
-18.600000000000005,
-10.399999999999999
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.012933889772125173,
"auroc_ci": [
-0.010202823134244223,
0.035816394946592514
],
"acc_diff": -13.4,
"acc_ci": [
-17.000000000000004,
-9.999999999999998
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.28702918242249087,
"auroc_ci": [
0.2568353566168098,
0.3170649500262892
],
"acc_diff": 4.71063257065949,
"acc_ci": [
2.8919919246298775,
6.596567967698515
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.020751028806584415,
"auroc_ci": [
-0.07973499767017543,
0.03798546250600538
],
"acc_diff": -9.145299145299147,
"acc_ci": [
-11.882478632478634,
-6.239316239316239
],
"n": 1170
}
},
"3b_w25_s1": {
"rjudge": {
"auroc_diff": 0.012753783684016362,
"auroc_ci": [
-0.0022454290142446347,
0.028200462410647744
],
"acc_diff": -0.3502626970227629,
"acc_ci": [
-2.8021015761821366,
2.1015761821365997
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.02844731753341989,
"auroc_ci": [
0.007618264488708323,
0.04923354143870089
],
"acc_diff": -12.059620596205967,
"acc_ci": [
-14.29539295392953,
-9.823848238482391
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.0327471788984407,
"auroc_ci": [
-0.05761304311407259,
-0.009461222572978614
],
"acc_diff": -9.599999999999998,
"acc_ci": [
-12.7,
-6.797500000000009
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": -0.0041119999999998935,
"auroc_ci": [
-0.0351643216308248,
0.024437785359221992
],
"acc_diff": -5.399999999999993,
"acc_ci": [
-8.604999999999995,
-2.0000000000000018
],
"n": 500
},
"atbench_claw": {
"auroc_diff": 0.009381624271330158,
"auroc_ci": [
-0.025415928547894248,
0.044345837307283746
],
"acc_diff": -10.799999999999999,
"acc_ci": [
-14.400000000000002,
-7.199999999999996
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.04804252782193952,
"auroc_ci": [
0.0250733041553122,
0.07301151646486885
],
"acc_diff": -10.999999999999998,
"acc_ci": [
-14.405000000000001,
-7.599999999999996
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.25969192870459556,
"auroc_ci": [
0.2292592215015387,
0.28952765482590154
],
"acc_diff": 0.0,
"acc_ci": [
-1.7496635262449491,
1.682368775235532
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.01669238683127572,
"auroc_ci": [
-0.0804397493653384,
0.04331673556690563
],
"acc_diff": 0.9401709401709368,
"acc_ci": [
-1.5384615384615332,
3.4188034188034178
],
"n": 1170
}
},
"3b_w25_s2": {
"rjudge": {
"auroc_diff": 0.021145564168820075,
"auroc_ci": [
0.006700309964832024,
0.03752507076019452
],
"acc_diff": 1.0507880910682998,
"acc_ci": [
-1.2259194395796924,
3.502626970227663
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.022828290377124194,
"auroc_ci": [
0.0031238371243867027,
0.042469731428552086
],
"acc_diff": -7.113821138211385,
"acc_ci": [
-9.214092140921403,
-4.945799457994582
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.04667368025248908,
"auroc_ci": [
-0.07252411100497831,
-0.02015684935744692
],
"acc_diff": -11.7,
"acc_ci": [
-15.000000000000002,
-8.599999999999996
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.007056000000000062,
"auroc_ci": [
-0.019189635999327423,
0.03317416873704169
],
"acc_diff": -3.1999999999999917,
"acc_ci": [
-6.199999999999994,
0.0
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.027441043985161673,
"auroc_ci": [
-0.06808204727298119,
0.012193757699612196
],
"acc_diff": -15.200000000000003,
"acc_ci": [
-19.599999999999994,
-11.2
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.04169978802331742,
"auroc_ci": [
0.01895329629011267,
0.06555202835571576
],
"acc_diff": -11.6,
"acc_ci": [
-15.200000000000003,
-8.199999999999996
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.2955433183323507,
"auroc_ci": [
0.26399779503821413,
0.32420600526109844
],
"acc_diff": 2.0188425302826385,
"acc_ci": [
0.40376850605652326,
3.6339165545087484
],
"n": 1486
},
"tracesafe": {
"auroc_diff": 0.02695987654320975,
"auroc_ci": [
-0.03407387644017941,
0.08863156776589111
],
"acc_diff": 4.61538461538461,
"acc_ci": [
2.051282051282055,
7.094017094017091
],
"n": 1170
}
},
"ens_3b": {
"rjudge": {
"auroc_diff": 0.014507198228128404,
"auroc_ci": [
0.0004782972336095232,
0.030349832756571066
],
"acc_diff": 1.0507880910682998,
"acc_ci": [
-1.2259194395796924,
3.3274956217162814
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.034791259913869776,
"auroc_ci": [
0.01425403115026993,
0.05458960308483688
],
"acc_diff": -10.70460704607047,
"acc_ci": [
-12.87262872628726,
-8.468834688346893
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.02957306463032683,
"auroc_ci": [
-0.05351164623209621,
-0.005439255831255008
],
"acc_diff": -9.699999999999998,
"acc_ci": [
-12.8,
-6.800000000000006
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.0154080000000002,
"auroc_ci": [
-0.01057826424070705,
0.04093074759841898
],
"acc_diff": -3.399999999999992,
"acc_ci": [
-6.599999999999994,
-0.19500000000001663
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.01888745363010058,
"auroc_ci": [
-0.05865248862753025,
0.01992588846920391
],
"acc_diff": -14.400000000000002,
"acc_ci": [
-18.599999999999994,
-10.599999999999998
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.03986983306836245,
"auroc_ci": [
0.01788477243650026,
0.0637672544746054
],
"acc_diff": -12.0,
"acc_ci": [
-15.600000000000003,
-8.599999999999996
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.287728908762395,
"auroc_ci": [
0.2569482310615392,
0.31657538489873904
],
"acc_diff": 2.0861372812920607,
"acc_ci": [
0.4037685060565288,
3.7685060565275874
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.0014660493827161059,
"auroc_ci": [
-0.06063631865899355,
0.056848360530981315
],
"acc_diff": -0.512820512820511,
"acc_ci": [
-2.991452991452992,
2.0512820512820444
],
"n": 1170
}
},
"3b_self": {
"rjudge": {
"auroc_diff": 0.015485418973791076,
"auroc_ci": [
0.0010110226351101649,
0.03146617492233492
],
"acc_diff": -1.4010507880910628,
"acc_ci": [
-4.0280210157618175,
1.2259194395796813
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.021624146402981936,
"auroc_ci": [
0.0005453722566786445,
0.04222668857474019
],
"acc_diff": -12.804878048780488,
"acc_ci": [
-15.176151761517609,
-10.433604336043356
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.03696333067990454,
"auroc_ci": [
-0.06175217559617585,
-0.01319974826585274
],
"acc_diff": -9.999999999999998,
"acc_ci": [
-13.0,
-7.100000000000007
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.010040000000000049,
"auroc_ci": [
-0.01861895740680239,
0.038350778012817964
],
"acc_diff": -3.599999999999992,
"acc_ci": [
-6.999999999999995,
-0.20000000000000018
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.09628378378378377,
"auroc_ci": [
-0.14448567028229925,
-0.04947067553895357
],
"acc_diff": -24.0,
"acc_ci": [
-28.800000000000004,
-19.399999999999995
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": -0.00780007949125594,
"auroc_ci": [
-0.0314053141176175,
0.014285791946421558
],
"acc_diff": -20.399999999999995,
"acc_ci": [
-24.6,
-16.400000000000002
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.29179067638615297,
"auroc_ci": [
0.2599722617447743,
0.32133444731801747
],
"acc_diff": 6.796769851951545,
"acc_ci": [
4.845222072678334,
8.613728129205922
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.017026748971193495,
"auroc_ci": [
-0.0769450072205232,
0.04312062155522243
],
"acc_diff": -8.37606837606838,
"acc_ci": [
-11.111111111111116,
-5.726495726495728
],
"n": 1170
}
},
"3b_self_s1": {
"rjudge": {
"auroc_diff": 0.017128091546696145,
"auroc_ci": [
0.0030576391508850137,
0.032383092001418035
],
"acc_diff": -2.276707530647981,
"acc_ci": [
-4.903677758318736,
0.17513134851139256
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.029203994507085662,
"auroc_ci": [
0.008684091523125268,
0.04879057133954991
],
"acc_diff": -13.821138211382122,
"acc_ci": [
-16.124661246612458,
-11.517615176151763
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.018476665159945838,
"auroc_ci": [
-0.04230980813574747,
0.004282228001457482
],
"acc_diff": -8.600000000000009,
"acc_ci": [
-11.50000000000001,
-5.700000000000005
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": -0.006735999999999964,
"auroc_ci": [
-0.03881613235338323,
0.02291633204942781
],
"acc_diff": -6.199999999999994,
"acc_ci": [
-9.605000000000008,
-2.800000000000002
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.020543521462639047,
"auroc_ci": [
-0.063630458433249,
0.020243752869270513
],
"acc_diff": -17.400000000000006,
"acc_ci": [
-21.605000000000008,
-13.200000000000001
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.06025602808691044,
"auroc_ci": [
0.03585688066715767,
0.08753958427271824
],
"acc_diff": -13.8,
"acc_ci": [
-17.400000000000006,
-10.199999999999998
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.2991953147091103,
"auroc_ci": [
0.26848260861523987,
0.3278932550243101
],
"acc_diff": 9.555854643337813,
"acc_ci": [
7.804508748317624,
11.305518169582768
],
"n": 1486
},
"tracesafe": {
"auroc_diff": -0.005612139917695491,
"auroc_ci": [
-0.06695983574010736,
0.054552536323508594
],
"acc_diff": -12.564102564102564,
"acc_ci": [
-15.470085470085465,
-9.829059829059828
],
"n": 1170
}
},
"3b_self_s2": {
"rjudge": {
"auroc_diff": 0.01711578688322868,
"auroc_ci": [
0.0029853125435418183,
0.03292415983176225
],
"acc_diff": -0.5253940455341444,
"acc_ci": [
-2.977232924693518,
1.9264448336252182
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.039857258961017106,
"auroc_ci": [
0.01945249695118748,
0.05975702002922339
],
"acc_diff": -13.008130081300816,
"acc_ci": [
-15.243902439024382,
-10.704607046070457
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.03744934817653445,
"auroc_ci": [
-0.06208566055056112,
-0.01261420272462027
],
"acc_diff": -8.600000000000009,
"acc_ci": [
-11.7,
-5.700000000000005
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.0011760000000000659,
"auroc_ci": [
-0.028211524855954493,
0.03049927508155185
],
"acc_diff": -5.599999999999994,
"acc_ci": [
-8.999999999999996,
-2.399999999999991
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.016833929517753088,
"auroc_ci": [
-0.05983320846856368,
0.022934508295743432
],
"acc_diff": -9.999999999999998,
"acc_ci": [
-13.805,
-6.399999999999995
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.05645535241123467,
"auroc_ci": [
0.03161485509370214,
0.08249296043373379
],
"acc_diff": -3.200000000000003,
"acc_ci": [
-5.800000000000005,
-0.6000000000000005
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.3022074928230123,
"auroc_ci": [
0.2709474530248363,
0.33088028199854513
],
"acc_diff": 7.806191117092864,
"acc_ci": [
5.854643337819648,
9.623149394347246
],
"n": 1486
},
"tracesafe": {
"auroc_diff": 0.022628600823045186,
"auroc_ci": [
-0.03683144915947646,
0.08031667119637309
],
"acc_diff": -5.8119658119658135,
"acc_ci": [
-8.632478632478623,
-3.1623931623931623
],
"n": 1170
}
},
"ens_3b_self": {
"rjudge": {
"auroc_diff": 0.018161683277962304,
"auroc_ci": [
0.004031585198545027,
0.0340638947780219
],
"acc_diff": -0.5253940455341444,
"acc_ci": [
-2.981611208406313,
2.1015761821365997
],
"n": 571
},
"assebench_strict": {
"auroc_diff": 0.034288676936299,
"auroc_ci": [
0.01404434534526663,
0.05431675893144182
],
"acc_diff": -12.533875338753397,
"acc_ci": [
-14.769647696476962,
-10.29810298102981
],
"n": 1476
},
"atbench": {
"auroc_diff": -0.02400086403110524,
"auroc_ci": [
-0.04726670989862959,
-0.0013094642967923812
],
"acc_diff": -8.600000000000009,
"acc_ci": [
-11.6,
-5.700000000000005
],
"n": 1000
},
"atbench_codex": {
"auroc_diff": 0.012032000000000154,
"auroc_ci": [
-0.01599360437117233,
0.03933030561738009
],
"acc_diff": -4.999999999999993,
"acc_ci": [
-8.199999999999996,
-1.6000000000000014
],
"n": 500
},
"atbench_claw": {
"auroc_diff": -0.03975390832008463,
"auroc_ci": [
-0.08410974086632922,
0.0020898519731997233
],
"acc_diff": -17.600000000000005,
"acc_ci": [
-21.999999999999996,
-13.395000000000028
],
"n": 500
},
"atbench_claw_native": {
"auroc_diff": 0.04831577901430839,
"auroc_ci": [
0.02534664119978466,
0.0732890359730846
],
"acc_diff": -13.200000000000001,
"acc_ci": [
-16.800000000000004,
-9.599999999999998
],
"n": 500
},
"agentdyn": {
"auroc_diff": 0.3020445428534456,
"auroc_ci": [
0.27128803419075315,
0.3308670754957477
],
"acc_diff": 6.998654104979806,
"acc_ci": [
5.248990578734858,
8.681022880215345
],
"n": 1486
},
"tracesafe": {
"auroc_diff": 0.0034413580246913655,
"auroc_ci": [
-0.05552856765928123,
0.06287506172146308
],
"acc_diff": -8.119658119658125,
"acc_ci": [
-10.85470085470085,
-5.470085470085473
],
"n": 1170
}
}
},
"claims": {
"U0_reproduced": {
"rjudge": true,
"atbench": true
},
"U1_lower_bounds": {
"atbench_codex": {
"h32_s0": 0.007647562328205493,
"h32_s1": -0.014216963202614254,
"h32_s2": -0.0017409265232811554
},
"atbench": {
"h32_s0": -0.03570004917010014,
"h32_s1": -0.040664058974714444,
"h32_s2": -0.045602033180343796
}
},
"U1_per_set": {
"atbench_codex": false,
"atbench": false
},
"U1_supported": false,
"U2_rjudge_lower_bounds_teacher_only": {
"teach_s0": 0.011744003450816496,
"teach_s1": 0.00915685507969467,
"teach_s2": 0.008199521078188624
},
"U2_all_positive": true,
"U2_seeds": 3
}
}
{
"sets": {
"rjudge": {
"n": 571,
"missing": 0,
"auroc": 0.9573151224314015,
"acc": 90.01751313485113,
"f1": 90.2229845626072,
"published": 90.4,
"reproduced": true
},
"assebench_strict": {
"n": 1476,
"missing": 0,
"auroc": 0.8310004016927144,
"acc": 79.53929539295393,
"f1": 79.45578231292517
},
"atbench": {
"n": 1000,
"missing": 0,
"auroc": 0.8438823797656717,
"acc": 78.3,
"f1": 77.41935483870968,
"published": 78.4,
"reproduced": true
},
"atbench_codex": {
"n": 500,
"missing": 0,
"auroc": 0.893264,
"acc": 83.2,
"f1": 84.21052631578947
},
"atbench_claw": {
"n": 500,
"missing": 0,
"auroc": 0.8878759273979862,
"acc": 87.6,
"f1": 89.8360655737705
},
"atbench_claw_native": {
"n": 500,
"missing": 0,
"auroc": 0.8906581213566508,
"acc": 88.8,
"f1": 90.81967213114754
},
"agentdyn": {
"n": 1486,
"missing": 4864,
"auroc": 0.6792305885848754,
"acc": 44.751009421265145,
"f1": 27.409372236958447
},
"tracesafe": {
"n": 1170,
"missing": 0,
"auroc": 0.5799948559670782,
"acc": 61.965811965811966,
"f1": 75.50908090258667,
"baa_0.5": 53.425925925925924
}
},
"vs_guards": {
"rjudge": {
"agentdog_4b": {
"auroc_diff": -0.009819121447028345,
"auroc_ci": [
-0.029417630933077036,
0.01036861132294604
],
"acc_diff": 1.5761821366024442,
"acc_ci": [
-1.5761821366024553,
4.728546409807355
],
"n": 571
},
"stepguard_4b": {
"auroc_diff": 0.024861572535991416,
"auroc_ci": [
0.0038273730049030066,
0.046898997689975214
],
"acc_diff": 0.5253940455341444,
"acc_ci": [
-1.7513134851138368,
2.977232924693518
],
"n": 571
},
"agentdog15_4b": {
"auroc_diff": -0.012919896640826933,
"auroc_ci": [
-0.027970378301906497,
0.001484768215849368
],
"acc_diff": -0.350262697022774,
"acc_ci": [