IntuitionGuard-8B / evaluation /h53_ens_w25.json
shreyaskapale's picture
IntuitionGuard-8B: three w25 seed adapters, model card, scorer, evaluation reports
3a88adf verified
Raw History Blame Contribute Delete
9.97 kB
{
"train": null,
"base": null,
"sets": {
"atbench_codex": {
"n": 500,
"auroc": 0.920176,
"acc": 81.39999999999999,
"vs_h32": {
"s0": {
"n": 500,
"ref": {
"auroc": 0.9259999999999999,
"acc": 81.2
},
"acc_diff": 0.19999999999998908,
"paired": {
"auroc_diff": -0.00582399999999994,
"auroc_ci": [
-0.017816605289016286,
0.0056432210649510085
],
"acc_diff": 0.19999999999998908,
"acc_ci": [
-1.8000000000000016,
2.200000000000002
]
}
},
"s1": {
"n": 500,
"ref": {
"auroc": 0.906984,
"acc": 76.8
},
"acc_diff": 4.5999999999999925,
"paired": {
"auroc_diff": 0.013191999999999982,
"auroc_ci": [
0.0036378436188693104,
0.02351186810063372
],
"acc_diff": 4.5999999999999925,
"acc_ci": [
2.400000000000002,
6.799999999999995
]
}
},
"s2": {
"n": 500,
"ref": {
"auroc": 0.91676,
"acc": 80.4
},
"acc_diff": 0.9999999999999898,
"paired": {
"auroc_diff": 0.0034159999999999746,
"auroc_ci": [
-0.0064309480885704285,
0.012835449918001162
],
"acc_diff": 0.9999999999999898,
"acc_ci": [
-0.40000000000000036,
2.400000000000002
]
}
}
}
},
"atbench": {
"n": 1000,
"auroc": 0.8341740302650896,
"acc": 71.3,
"vs_h32": {
"s0": {
"n": 1000,
"ref": {
"auroc": 0.8317159417739037,
"acc": 73.4
},
"acc_diff": -2.100000000000002,
"paired": {
"auroc_diff": 0.0024580884911858947,
"auroc_ci": [
-0.006278397607167122,
0.011470272398567382
],
"acc_diff": -2.100000000000002,
"acc_ci": [
-4.200000000000004,
0.0
]
}
},
"s1": {
"n": 1000,
"ref": {
"auroc": 0.8275257909284734,
"acc": 69.39999999999999
},
"acc_diff": 1.9000000000000017,
"paired": {
"auroc_diff": 0.00664823933661618,
"auroc_ci": [
-0.003232769051501072,
0.01610391290726812
],
"acc_diff": 1.9000000000000017,
"acc_ci": [
0.20000000000000018,
3.600000000000003
]
}
},
"s2": {
"n": 1000,
"ref": {
"auroc": 0.8234456440431857,
"acc": 72.89999999999999
},
"acc_diff": -1.6000000000000014,
"paired": {
"auroc_diff": 0.010728386221903974,
"auroc_ci": [
-0.0003375654409279032,
0.02204627048786279
],
"acc_diff": -1.6000000000000014,
"acc_ci": [
-3.6025000000000027,
0.40000000000000036
]
}
}
}
},
"rjudge": {
"n": 571,
"auroc": 0.979328165374677,
"acc": 92.29422066549913,
"vs_h32": {
"s0": {
"n": 571,
"ref": {
"auroc": 0.9651901070505722,
"acc": 87.56567425569177
},
"acc_diff": 4.728546409807355
},
"s1": {
"n": 571,
"ref": {
"auroc": 0.9761535622000739,
"acc": 91.41856392294221
},
"acc_diff": 0.8756567425569184
},
"s2": {
"n": 571,
"ref": {
"auroc": 0.9654792666420573,
"acc": 89.66725043782837
},
"acc_diff": 2.626970227670755
}
}
},
"assebench_strict": {
"n": 1476,
"auroc": 0.8805029566452119,
"acc": 71.61246612466124,
"vs_h32": {
"s0": {
"n": 1476,
"ref": {
"auroc": 0.8654021130905116,
"acc": 65.5149051490515
},
"acc_diff": 6.097560975609751
},
"s1": {
"n": 1476,
"ref": {
"auroc": 0.86411015722066,
"acc": 68.4959349593496
},
"acc_diff": 3.1165311653116534
},
"s2": {
"n": 1476,
"ref": {
"auroc": 0.8685138303735742,
"acc": 69.10569105691057
},
"acc_diff": 2.5067750677506773
}
}
},
"atbench_claw_native": {
"n": 500,
"auroc": 0.9208233969263382,
"acc": 78.8,
"vs_h32": {
"s0": {
"n": 500,
"ref": {
"auroc": 0.8973320747217807,
"acc": 76.6
},
"acc_diff": 2.200000000000002
},
"s1": {
"n": 500,
"ref": {
"auroc": 0.9220488871224166,
"acc": 75.2
},
"acc_diff": 3.600000000000003
},
"s2": {
"n": 500,
"ref": {
"auroc": 0.9198877186009539,
"acc": 79.80000000000001
},
"acc_diff": -1.0000000000000009
}
}
},
"agentdyn": {
"n": 1492,
"auroc": 0.957038885620481,
"acc": 50.26809651474531,
"vs_h32": {
"s0": {
"n": 1492,
"ref": {
"auroc": 0.975573131155941,
"acc": 69.10187667560321
},
"acc_diff": -18.833780160857906
},
"s1": {
"n": 1492,
"ref": {
"auroc": 0.971545081889435,
"acc": 57.171581769437
},
"acc_diff": -6.903485254691688
},
"s2": {
"n": 1492,
"ref": {
"auroc": 0.9655884083274818,
"acc": 62.13136729222521
},
"acc_diff": -11.863270777479896
}
}
},
"tracesafe": {
"n": 1170,
"auroc": 0.5497582304526749,
"acc": 56.837606837606835,
"baa_0.5": 54.72222222222223,
"vs_h32": {
"s0": {
"n": 1170,
"ref": {
"auroc": 0.5774228395061729,
"acc": 47.008547008547005
},
"acc_diff": 9.829059829059828
},
"s1": {
"n": 1170,
"ref": {
"auroc": 0.5978343621399177,
"acc": 56.58119658119658
},
"acc_diff": 0.2564102564102555
},
"s2": {
"n": 1170,
"ref": {
"auroc": 0.591059670781893,
"acc": 47.521367521367516
},
"acc_diff": 9.316239316239317
}
}
}
},
"latency_p50_ms": {},
"X4_calibrated_cut": {
"tau": 0.68,
"calibration": {
"n": 699,
"balanced_acc_tau": 95.85973962171457,
"balanced_acc_0.5": 93.33210513387374
},
"sets": {
"rjudge": {
"n": 571,
"acc_0.5": 92.29422066549913,
"acc_tau": 89.49211908931699,
"f1_0.5": 92.226148409894,
"f1_tau": 89.01098901098901,
"oracle_tau": 0.27,
"oracle_acc": 94.04553415061297
},
"assebench_strict": {
"n": 1476,
"acc_0.5": 71.61246612466124,
"acc_tau": 64.63414634146342,
"f1_0.5": 68.66118175018698,
"f1_tau": 55.61224489795919,
"oracle_tau": 0.1,
"oracle_acc": 80.62330623306234
},
"atbench": {
"n": 1000,
"acc_0.5": 71.3,
"acc_tau": 76.1,
"f1_0.5": 74.3978590544157,
"f1_tau": 75.97989949748744,
"oracle_tau": 0.74,
"oracle_acc": 78.7
},
"atbench_codex": {
"n": 500,
"acc_0.5": 81.39999999999999,
"acc_tau": 83.39999999999999,
"f1_0.5": 83.99311531841653,
"f1_tau": 85.1520572450805,
"oracle_tau": 0.67,
"oracle_acc": 83.6
}
}
},
"claims": {
"X1_per_set": {
"atbench_codex": [
-0.017816605289016286,
0.0056432210649510085
],
"atbench": [
-0.006278397607167122,
0.011470272398567382
]
},
"X1_supported": false,
"X2_acc": {
"rjudge": 92.29422066549913,
"atbench": 71.3
},
"X2_bar": {
"rjudge": 90.4,
"atbench": 72.4
},
"X2_supported": false,
"X3_vs_seeds_1_2": {
"atbench_codex": {
"s1": [
0.0036378436188693104,
0.02351186810063372
],
"s2": [
-0.0064309480885704285,
0.012835449918001162
]
},
"atbench": {
"s1": [
-0.003232769051501072,
0.01610391290726812
],
"s2": [
-0.0003375654409279032,
0.02204627048786279
]
}
}
},
"vs_guards": {
"atbench_codex": {
"agentdog_4b": {
"auroc_diff": 0.08737600000000001,
"auroc_ci": [
0.05892936392245434,
0.11643739146055611
],
"acc_diff": 1.19999999999999,
"acc_ci": [
-1.999999999999991,
4.399999999999993
],
"n": 500
},
"stepguard_4b": {
"auroc_diff": 0.07766400000000007,
"auroc_ci": [
0.04947533703764029,
0.1067245206917969
],
"acc_diff": 3.1999999999999917,
"acc_ci": [
-0.20000000000000018,
6.799999999999995
],
"n": 500
},
"agentdog15_4b": {
"auroc_diff": 0.04672942783278855,
"auroc_ci": [
0.021788150792624035,
0.07281198309581959
],
"acc_diff": 3.8306451612903247,
"acc_ci": [
1.0080645161290258,
6.854838709677424
],
"n": 496
}
},
"atbench": {
"agentdog_4b": {
"auroc_diff": 0.2316283386201905,
"auroc_ci": [
0.20211449136459286,
0.26244215874280413
],
"acc_diff": 11.099999999999998,
"acc_ci": [
8.200000000000006,
13.900000000000002
],
"n": 1000
},
"stepguard_4b": {
"auroc_diff": 0.04670168126052543,
"auroc_ci": [
0.018970855969553564,
0.07253321422256387
],
"acc_diff": -1.7000000000000015,
"acc_ci": [
-4.902500000000003,
1.6000000000000014
],
"n": 1000
},
"agentdog15_4b": {
"auroc_diff": 0.0910115919724459,
"auroc_ci": [
0.06327126043629389,
0.11848111068178413
],
"acc_diff": 3.549695740365111,
"acc_ci": [
0.40567951318457585,
6.592292089249496
],
"n": 986
}
},
"rjudge": {
"agentdog_4b": {
"auroc_diff": 0.012193921496247206,
"auroc_ci": [
-0.0013734734750716765,
0.027566819235545093
],
"acc_diff": 3.8528896672504365,
"acc_ci": [
0.5210157618213603,
7.005253940455347
],
"n": 571
},
"stepguard_4b": {
"auroc_diff": 0.04687461547926697,
"auroc_ci": [
0.029323098673131785,
0.0656522129420753
],
"acc_diff": 2.8021015761821366,
"acc_ci": [
0.5253940455341444,
5.078809106830128
],
"n": 571
},
"agentdog15_4b": {
"auroc_diff": 0.009093146302448618,
"auroc_ci": [
0.0010304723577039265,
0.017428346840814703
],
"acc_diff": 1.9264448336252182,
"acc_ci": [
-0.5253940455341444,
4.20315236427321
],
"n": 571
}
}
}
}