{ "train": null, "base": null, "sets": { "atbench_codex": { "n": 500, "auroc": 0.920176, "acc": 81.39999999999999, "vs_h32": { "s0": { "n": 500, "ref": { "auroc": 0.9259999999999999, "acc": 81.2 }, "acc_diff": 0.19999999999998908, "paired": { "auroc_diff": -0.00582399999999994, "auroc_ci": [ -0.017816605289016286, 0.0056432210649510085 ], "acc_diff": 0.19999999999998908, "acc_ci": [ -1.8000000000000016, 2.200000000000002 ] } }, "s1": { "n": 500, "ref": { "auroc": 0.906984, "acc": 76.8 }, "acc_diff": 4.5999999999999925, "paired": { "auroc_diff": 0.013191999999999982, "auroc_ci": [ 0.0036378436188693104, 0.02351186810063372 ], "acc_diff": 4.5999999999999925, "acc_ci": [ 2.400000000000002, 6.799999999999995 ] } }, "s2": { "n": 500, "ref": { "auroc": 0.91676, "acc": 80.4 }, "acc_diff": 0.9999999999999898, "paired": { "auroc_diff": 0.0034159999999999746, "auroc_ci": [ -0.0064309480885704285, 0.012835449918001162 ], "acc_diff": 0.9999999999999898, "acc_ci": [ -0.40000000000000036, 2.400000000000002 ] } } } }, "atbench": { "n": 1000, "auroc": 0.8341740302650896, "acc": 71.3, "vs_h32": { "s0": { "n": 1000, "ref": { "auroc": 0.8317159417739037, "acc": 73.4 }, "acc_diff": -2.100000000000002, "paired": { "auroc_diff": 0.0024580884911858947, "auroc_ci": [ -0.006278397607167122, 0.011470272398567382 ], "acc_diff": -2.100000000000002, "acc_ci": [ -4.200000000000004, 0.0 ] } }, "s1": { "n": 1000, "ref": { "auroc": 0.8275257909284734, "acc": 69.39999999999999 }, "acc_diff": 1.9000000000000017, "paired": { "auroc_diff": 0.00664823933661618, "auroc_ci": [ -0.003232769051501072, 0.01610391290726812 ], "acc_diff": 1.9000000000000017, "acc_ci": [ 0.20000000000000018, 3.600000000000003 ] } }, "s2": { "n": 1000, "ref": { "auroc": 0.8234456440431857, "acc": 72.89999999999999 }, "acc_diff": -1.6000000000000014, "paired": { "auroc_diff": 0.010728386221903974, "auroc_ci": [ -0.0003375654409279032, 0.02204627048786279 ], "acc_diff": -1.6000000000000014, "acc_ci": [ -3.6025000000000027, 0.40000000000000036 ] } } } }, "rjudge": { "n": 571, "auroc": 0.979328165374677, "acc": 92.29422066549913, "vs_h32": { "s0": { "n": 571, "ref": { "auroc": 0.9651901070505722, "acc": 87.56567425569177 }, "acc_diff": 4.728546409807355 }, "s1": { "n": 571, "ref": { "auroc": 0.9761535622000739, "acc": 91.41856392294221 }, "acc_diff": 0.8756567425569184 }, "s2": { "n": 571, "ref": { "auroc": 0.9654792666420573, "acc": 89.66725043782837 }, "acc_diff": 2.626970227670755 } } }, "assebench_strict": { "n": 1476, "auroc": 0.8805029566452119, "acc": 71.61246612466124, "vs_h32": { "s0": { "n": 1476, "ref": { "auroc": 0.8654021130905116, "acc": 65.5149051490515 }, "acc_diff": 6.097560975609751 }, "s1": { "n": 1476, "ref": { "auroc": 0.86411015722066, "acc": 68.4959349593496 }, "acc_diff": 3.1165311653116534 }, "s2": { "n": 1476, "ref": { "auroc": 0.8685138303735742, "acc": 69.10569105691057 }, "acc_diff": 2.5067750677506773 } } }, "atbench_claw_native": { "n": 500, "auroc": 0.9208233969263382, "acc": 78.8, "vs_h32": { "s0": { "n": 500, "ref": { "auroc": 0.8973320747217807, "acc": 76.6 }, "acc_diff": 2.200000000000002 }, "s1": { "n": 500, "ref": { "auroc": 0.9220488871224166, "acc": 75.2 }, "acc_diff": 3.600000000000003 }, "s2": { "n": 500, "ref": { "auroc": 0.9198877186009539, "acc": 79.80000000000001 }, "acc_diff": -1.0000000000000009 } } }, "agentdyn": { "n": 1492, "auroc": 0.957038885620481, "acc": 50.26809651474531, "vs_h32": { "s0": { "n": 1492, "ref": { "auroc": 0.975573131155941, "acc": 69.10187667560321 }, "acc_diff": -18.833780160857906 }, "s1": { "n": 1492, "ref": { "auroc": 0.971545081889435, "acc": 57.171581769437 }, "acc_diff": -6.903485254691688 }, "s2": { "n": 1492, "ref": { "auroc": 0.9655884083274818, "acc": 62.13136729222521 }, "acc_diff": -11.863270777479896 } } }, "tracesafe": { "n": 1170, "auroc": 0.5497582304526749, "acc": 56.837606837606835, "baa_0.5": 54.72222222222223, "vs_h32": { "s0": { "n": 1170, "ref": { "auroc": 0.5774228395061729, "acc": 47.008547008547005 }, "acc_diff": 9.829059829059828 }, "s1": { "n": 1170, "ref": { "auroc": 0.5978343621399177, "acc": 56.58119658119658 }, "acc_diff": 0.2564102564102555 }, "s2": { "n": 1170, "ref": { "auroc": 0.591059670781893, "acc": 47.521367521367516 }, "acc_diff": 9.316239316239317 } } } }, "latency_p50_ms": {}, "X4_calibrated_cut": { "tau": 0.68, "calibration": { "n": 699, "balanced_acc_tau": 95.85973962171457, "balanced_acc_0.5": 93.33210513387374 }, "sets": { "rjudge": { "n": 571, "acc_0.5": 92.29422066549913, "acc_tau": 89.49211908931699, "f1_0.5": 92.226148409894, "f1_tau": 89.01098901098901, "oracle_tau": 0.27, "oracle_acc": 94.04553415061297 }, "assebench_strict": { "n": 1476, "acc_0.5": 71.61246612466124, "acc_tau": 64.63414634146342, "f1_0.5": 68.66118175018698, "f1_tau": 55.61224489795919, "oracle_tau": 0.1, "oracle_acc": 80.62330623306234 }, "atbench": { "n": 1000, "acc_0.5": 71.3, "acc_tau": 76.1, "f1_0.5": 74.3978590544157, "f1_tau": 75.97989949748744, "oracle_tau": 0.74, "oracle_acc": 78.7 }, "atbench_codex": { "n": 500, "acc_0.5": 81.39999999999999, "acc_tau": 83.39999999999999, "f1_0.5": 83.99311531841653, "f1_tau": 85.1520572450805, "oracle_tau": 0.67, "oracle_acc": 83.6 } } }, "claims": { "X1_per_set": { "atbench_codex": [ -0.017816605289016286, 0.0056432210649510085 ], "atbench": [ -0.006278397607167122, 0.011470272398567382 ] }, "X1_supported": false, "X2_acc": { "rjudge": 92.29422066549913, "atbench": 71.3 }, "X2_bar": { "rjudge": 90.4, "atbench": 72.4 }, "X2_supported": false, "X3_vs_seeds_1_2": { "atbench_codex": { "s1": [ 0.0036378436188693104, 0.02351186810063372 ], "s2": [ -0.0064309480885704285, 0.012835449918001162 ] }, "atbench": { "s1": [ -0.003232769051501072, 0.01610391290726812 ], "s2": [ -0.0003375654409279032, 0.02204627048786279 ] } } }, "vs_guards": { "atbench_codex": { "agentdog_4b": { "auroc_diff": 0.08737600000000001, "auroc_ci": [ 0.05892936392245434, 0.11643739146055611 ], "acc_diff": 1.19999999999999, "acc_ci": [ -1.999999999999991, 4.399999999999993 ], "n": 500 }, "stepguard_4b": { "auroc_diff": 0.07766400000000007, "auroc_ci": [ 0.04947533703764029, 0.1067245206917969 ], "acc_diff": 3.1999999999999917, "acc_ci": [ -0.20000000000000018, 6.799999999999995 ], "n": 500 }, "agentdog15_4b": { "auroc_diff": 0.04672942783278855, "auroc_ci": [ 0.021788150792624035, 0.07281198309581959 ], "acc_diff": 3.8306451612903247, "acc_ci": [ 1.0080645161290258, 6.854838709677424 ], "n": 496 } }, "atbench": { "agentdog_4b": { "auroc_diff": 0.2316283386201905, "auroc_ci": [ 0.20211449136459286, 0.26244215874280413 ], "acc_diff": 11.099999999999998, "acc_ci": [ 8.200000000000006, 13.900000000000002 ], "n": 1000 }, "stepguard_4b": { "auroc_diff": 0.04670168126052543, "auroc_ci": [ 0.018970855969553564, 0.07253321422256387 ], "acc_diff": -1.7000000000000015, "acc_ci": [ -4.902500000000003, 1.6000000000000014 ], "n": 1000 }, "agentdog15_4b": { "auroc_diff": 0.0910115919724459, "auroc_ci": [ 0.06327126043629389, 0.11848111068178413 ], "acc_diff": 3.549695740365111, "acc_ci": [ 0.40567951318457585, 6.592292089249496 ], "n": 986 } }, "rjudge": { "agentdog_4b": { "auroc_diff": 0.012193921496247206, "auroc_ci": [ -0.0013734734750716765, 0.027566819235545093 ], "acc_diff": 3.8528896672504365, "acc_ci": [ 0.5210157618213603, 7.005253940455347 ], "n": 571 }, "stepguard_4b": { "auroc_diff": 0.04687461547926697, "auroc_ci": [ 0.029323098673131785, 0.0656522129420753 ], "acc_diff": 2.8021015761821366, "acc_ci": [ 0.5253940455341444, 5.078809106830128 ], "n": 571 }, "agentdog15_4b": { "auroc_diff": 0.009093146302448618, "auroc_ci": [ 0.0010304723577039265, 0.017428346840814703 ], "acc_diff": 1.9264448336252182, "acc_ci": [ -0.5253940455341444, 4.20315236427321 ], "n": 571 } } } }