{ "primary": "ens_w25", "ens_w25": { "E1": { "rjudge_acc": 92.29422066549913, "acc_ok": true, "rjudge_lower_bounds": { "agentdog_4b": -0.0013734734750716765, "stepguard_4b": 0.029323098673131785, "agentdog15_4b": 0.0010304723577039265, "agentdog15u_4b": 0.007454665057679738 }, "lb_ok": false, "supported": false }, "E1_supported": false, "E2": { "older_lower_bounds": { "atbench_codex": { "agentdog_4b": 0.05892936392245434, "stepguard_4b": 0.04947533703764029, "agentdog15_4b": 0.021788150792624035 }, "atbench": { "agentdog_4b": 0.20211449136459286, "stepguard_4b": 0.018970855969553564, "agentdog15_4b": 0.06327126043629389 } }, "older_ok": true, "unified_lower_bounds": { "atbench_codex": 0.0015388753759192797, "atbench": -0.03346510544943757 }, "unified_ok": false, "supported": false }, "E2_supported": false, "E3_vs_members": { "h47_w25": { "rjudge": { "auroc_diff": -0.005284852959271502, "auroc_ci": [ -0.010059185460479866, -0.0014848727557587366 ], "acc_diff": 1.4010507880910739, "acc_ci": [ 0.17513134851138146, 2.8021015761821366 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.003027642063766378, "auroc_ci": [ -0.0015410766860840142, 0.007334602350236927 ], "acc_diff": 5.555555555555558, "acc_ci": [ 3.99728997289972, 6.978319783197828 ], "n": 1476 }, "atbench": { "auroc_diff": 0.011528415022940997, "auroc_ci": [ 0.005455408301921749, 0.01806280026080941 ], "acc_diff": -0.6000000000000005, "acc_ci": [ -2.100000000000002, 0.8000000000000007 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.0021919999999999717, "auroc_ci": [ -0.004930372148859654, 0.009832481910421239 ], "acc_diff": -1.4000000000000012, "acc_ci": [ -3.1999999999999917, 0.20000000000000018 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.010052331743507947, "auroc_ci": [ -0.01673495840097399, -0.0039021250812851004 ], "acc_diff": -1.8000000000000016, "acc_ci": [ -3.400000000000003, 0.0 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.006235095389507173, "auroc_ci": [ 0.0007612951689983882, 0.012286809758806755 ], "acc_diff": -2.400000000000002, "acc_ci": [ -3.999999999999993, -1.0000000000000009 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.001878807431727303, "auroc_ci": [ -0.0017563795015893197, 0.0054734010877586494 ], "acc_diff": -4.6246648793565655, "acc_ci": [ -5.76575067024129, -3.4852546916890104 ], "n": 1492 }, "tracesafe": { "auroc_diff": 0.025082304526748977, "auroc_ci": [ 0.013332229296844003, 0.036440131714432986 ], "acc_diff": 10.51282051282051, "acc_ci": [ 8.71794871794872, 12.393162393162394 ], "n": 1170 } }, "h47_w25_s1": { "rjudge": { "auroc_diff": 0.000584471514704088, "auroc_ci": [ -0.0021191789417152164, 0.0032065449460079733 ], "acc_diff": 0.7005253940455369, "acc_ci": [ -0.17513134851138173, 1.7513134851138368 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.009028744383308251, "auroc_ci": [ 0.004920956467950491, 0.013237151019299728 ], "acc_diff": -1.1517615176151796, "acc_ci": [ -2.4390243902438935, 0.06775067750678376 ], "n": 1476 }, "atbench": { "auroc_diff": 0.010758387301942895, "auroc_ci": [ 0.003837955337748994, 0.018051716306787043 ], "acc_diff": 3.399999999999992, "acc_ci": [ 1.8000000000000016, 4.799999999999994 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.015759999999999885, "auroc_ci": [ 0.005816674787243778, 0.026278096368262335 ], "acc_diff": 3.399999999999992, "acc_ci": [ 1.200000000000001, 5.799999999999994 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.0005879040805509828, "auroc_ci": [ -0.00580451975174969, 0.0047174544797580755 ], "acc_diff": 0.40000000000000036, "acc_ci": [ -1.0000000000000009, 1.8000000000000016 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": -0.006872681505034284, "auroc_ci": [ -0.012844343817324508, -0.0012137829208492287 ], "acc_diff": 0.0, "acc_ci": [ -1.4000000000000012, 1.4000000000000012 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.0035939308826600236, "auroc_ci": [ -0.0008863418049177868, 0.008055351347295408 ], "acc_diff": 4.490616621983912, "acc_ci": [ 3.418230563002683, 5.5646782841823 ], "n": 1492 }, "tracesafe": { "auroc_diff": -0.01300925925925922, "auroc_ci": [ -0.024927965828244792, -0.0021736043481458073 ], "acc_diff": -2.3931623931623958, "acc_ci": [ -3.589743589743588, -1.1965811965812034 ], "n": 1170 } }, "h47_w25_s2": { "rjudge": { "auroc_diff": 0.006829088224437063, "auroc_ci": [ 0.003652745283536635, 0.01100189196511411 ], "acc_diff": -0.5253940455341444, "acc_ci": [ -1.7513134851138368, 0.7005253940455369 ], "n": 571 }, "assebench_strict": { "auroc_diff": -0.00132745429577652, "auroc_ci": [ -0.0051212480511259265, 0.0025766856455846665 ], "acc_diff": -3.7262872628726296, "acc_ci": [ -5.0135501355013545, -2.5067750677506773 ], "n": 1476 }, "atbench": { "auroc_diff": -0.0014900536419309995, "auroc_ci": [ -0.00945462213551489, 0.006717971747974117 ], "acc_diff": -1.7000000000000015, "acc_ci": [ -3.600000000000003, 0.10000000000000009 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.008704000000000045, "auroc_ci": [ 0.0008265359914208126, 0.016545292007456247 ], "acc_diff": 0.39999999999998925, "acc_ci": [ -1.200000000000001, 2.0000000000000018 ], "n": 500 }, "atbench_claw": { "auroc_diff": 0.018026298357180948, "auroc_ci": [ 0.011566190327626036, 0.025595121563466317 ], "acc_diff": 4.400000000000004, "acc_ci": [ 2.6000000000000023, 6.599999999999994 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.006094329623741568, "auroc_ci": [ 0.0007635358087123651, 0.011894740479757993 ], "acc_diff": 2.200000000000002, "acc_ci": [ 0.6000000000000005, 3.8000000000000034 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.007358662440931529, "auroc_ci": [ 0.002699004213621195, 0.011965268439070365 ], "acc_diff": 0.6032171581769441, "acc_ci": [ -0.6702412868632712, 1.8766756032171594 ], "n": 1492 }, "tracesafe": { "auroc_diff": -0.01588991769547321, "auroc_ci": [ -0.03180771704911614, 0.00038553689084559783 ], "acc_diff": -8.119658119658125, "acc_ci": [ -9.914529914529924, -6.41025641025641 ], "n": 1170 } } }, "E3_atbench_acc": { "ours": 71.3, "unified": 78.3 }, "sets": { "atbench_codex": [ 0.92, 81.4 ], "atbench": [ 0.834, 71.3 ], "rjudge": [ 0.979, 92.3 ], "assebench_strict": [ 0.881, 71.6 ], "atbench_claw_native": [ 0.921, 78.8 ], "agentdyn": [ 0.957, 50.3 ], "tracesafe": [ 0.55, 56.8 ] }, "X4": { "rjudge": [ 92.3, 89.5 ], "assebench_strict": [ 71.6, 64.6 ], "atbench": [ 71.3, 76.1 ], "atbench_codex": [ 81.4, 83.4 ], "tau": 0.68 }, "vs_unified_all_sets": { "rjudge": [ 0.022, [ 0.007, 0.038 ], 2.3 ], "assebench_strict": [ 0.05, [ 0.03, 0.068 ], -7.9 ], "atbench": [ -0.01, [ -0.033, 0.013 ], -7.0 ], "atbench_codex": [ 0.027, [ 0.002, 0.051 ], -1.8 ], "atbench_claw": [ -0.056, [ -0.1, -0.013 ], -13.4 ], "atbench_claw_native": [ 0.03, [ 0.008, 0.054 ], -10.0 ], "agentdyn": [ 0.283, [ 0.254, 0.312 ], 5.4 ], "tracesafe": [ -0.03, [ -0.089, 0.027 ], -5.1 ] } }, "secondary": { "ens5": { "E1": { "rjudge_acc": 91.59369527145358, "acc_ok": false, "rjudge_lower_bounds": { "agentdog_4b": -0.002909216886524685, "stepguard_4b": 0.028578315519013235, "agentdog15_4b": -0.0001369869679280849, "agentdog15u_4b": 0.006367251053043199 }, "lb_ok": false, "supported": false }, "E1_supported": false, "E2": { "older_lower_bounds": { "atbench_codex": { "agentdog_4b": 0.0626968420617803, "stepguard_4b": 0.053074898453990944, "agentdog15_4b": 0.024911228480834432 }, "atbench": { "agentdog_4b": 0.20814610030183622, "stepguard_4b": 0.02431795800013558, "agentdog15_4b": 0.06886326977249825 } }, "older_ok": true, "unified_lower_bounds": { "atbench_codex": 0.006398239792704952, "atbench": -0.027443739316762052 }, "unified_ok": true, "supported": true }, "E2_supported": true, "E3_vs_members": { "h47_w25": { "rjudge": { "auroc_diff": -0.006330749354005127, "auroc_ci": [ -0.011876660618888837, -0.0017107997272030428 ], "acc_diff": 0.7005253940455369, "acc_ci": [ -0.5253940455341555, 1.9264448336252182 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.002510112380543106, "auroc_ci": [ -0.0023123165766014836, 0.007215025540276576 ], "acc_diff": 5.216802168021683, "acc_ci": [ 3.5907859078590842, 6.707317073170738 ], "n": 1476 }, "atbench": { "auroc_diff": 0.016600597621514446, "auroc_ci": [ 0.009559169675140883, 0.023651034324689065 ], "acc_diff": 0.6000000000000005, "acc_ci": [ -0.8000000000000007, 2.0000000000000018 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.005487999999999937, "auroc_ci": [ -0.002743211757248151, 0.014074652375331446 ], "acc_diff": -1.200000000000001, "acc_ci": [ -2.8000000000000025, 0.40000000000000036 ], "n": 500 }, "atbench_claw": { "auroc_diff": 0.01796833598304204, "auroc_ci": [ 0.009900354220907426, 0.02634001025794135 ], "acc_diff": -1.4000000000000012, "acc_ci": [ -3.0000000000000027, 0.20000000000000018 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.013240262321144614, "auroc_ci": [ 0.006486473671891271, 0.020573326164219884 ], "acc_diff": -1.0000000000000009, "acc_ci": [ -2.6000000000000023, 0.6000000000000005 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.02135008445144526, "auroc_ci": [ 0.015821558069750104, 0.027422890890859093 ], "acc_diff": -3.686327077747986, "acc_ci": [ -4.825737265415553, -2.6139410187667522 ], "n": 1492 }, "tracesafe": { "auroc_diff": 0.03453189300411519, "auroc_ci": [ 0.02117174299565245, 0.047650382775224404 ], "acc_diff": 11.36752136752136, "acc_ci": [ 9.487179487179487, 13.418803418803416 ], "n": 1170 } }, "h47_w25_s1": { "rjudge": { "auroc_diff": -0.0004614248800295373, "auroc_ci": [ -0.00381655234288735, 0.0027222216155852255 ], "acc_diff": 0.0, "acc_ci": [ -1.050788091068311, 1.2259194395796813 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.00851121470008498, "auroc_ci": [ 0.003886815527316559, 0.013401982555323324 ], "acc_diff": -1.490514905149054, "acc_ci": [ -2.8455284552845517, -0.20155826558265819 ], "n": 1476 }, "atbench": { "auroc_diff": 0.015830569900516345, "auroc_ci": [ 0.008542068273074768, 0.023317723083569324 ], "acc_diff": 4.5999999999999925, "acc_ci": [ 2.8000000000000025, 6.300000000000006 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.01905599999999985, "auroc_ci": [ 0.007813964269156903, 0.030694744246315266 ], "acc_diff": 3.599999999999992, "acc_ci": [ 1.4000000000000012, 5.800000000000005 ], "n": 500 }, "atbench_claw": { "auroc_diff": 0.027432763645999003, "auroc_ci": [ 0.01882741527586003, 0.03668695515422081 ], "acc_diff": 0.8000000000000007, "acc_ci": [ -0.40000000000000036, 2.0000000000000018 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.0001324854266031572, "auroc_ci": [ -0.006177333728409717, 0.005971725340305829 ], "acc_diff": 1.4000000000000012, "acc_ci": [ 0.0, 2.8000000000000025 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.023065207902377982, "auroc_ci": [ 0.016376139372327278, 0.03020523858652195 ], "acc_diff": 5.428954423592492, "acc_ci": [ 4.2895442359249305, 6.702412868632707 ], "n": 1492 }, "tracesafe": { "auroc_diff": -0.0035596707818930096, "auroc_ci": [ -0.016239129779048657, 0.009120657673918094 ], "acc_diff": -1.538461538461544, "acc_ci": [ -2.8226495726495733, -0.34188034188034067 ], "n": 1170 } }, "h47_w25_s2": { "rjudge": { "auroc_diff": 0.005783191829703438, "auroc_ci": [ 0.0028046768163890318, 0.009604772603822177 ], "acc_diff": -1.2259194395796813, "acc_ci": [ -2.626970227670755, 0.0 ], "n": 571 }, "assebench_strict": { "auroc_diff": -0.0018449839789997924, "auroc_ci": [ -0.005946035458135851, 0.0024127048210468485 ], "acc_diff": -4.0650406504065035, "acc_ci": [ -5.216802168021683, -2.8455284552845517 ], "n": 1476 }, "atbench": { "auroc_diff": 0.0035821289566424497, "auroc_ci": [ -0.005367769121692692, 0.012382617151886417 ], "acc_diff": -0.5000000000000004, "acc_ci": [ -2.200000000000002, 1.200000000000001 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.01200000000000001, "auroc_ci": [ 0.003800953797565285, 0.02028843599302695 ], "acc_diff": 0.5999999999999894, "acc_ci": [ -1.19999999999999, 2.200000000000002 ], "n": 500 }, "atbench_claw": { "auroc_diff": 0.046046966083730934, "auroc_ci": [ 0.0351332552193211, 0.05819549120120565 ], "acc_diff": 4.800000000000004, "acc_ci": [ 2.8000000000000025, 6.999999999999995 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.013099496555379009, "auroc_ci": [ 0.0069686951833785316, 0.020117551422288903 ], "acc_diff": 3.600000000000003, "acc_ci": [ 1.8000000000000016, 5.600000000000005 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.026829939460649488, "auroc_ci": [ 0.01978051613973936, 0.0347806416036584 ], "acc_diff": 1.5415549597855238, "acc_ci": [ 0.2680965147453085, 2.7479892761394122 ], "n": 1492 }, "tracesafe": { "auroc_diff": -0.006440329218106999, "auroc_ci": [ -0.022666747570811746, 0.010422588483751102 ], "acc_diff": -7.264957264957273, "acc_ci": [ -9.059829059829061, -5.467948717948721 ], "n": 1170 } }, "h51_w25u": { "rjudge": { "auroc_diff": 0.011972437553832926, "auroc_ci": [ 0.0060371659565142035, 0.019712795121644835 ], "acc_diff": 0.5253940455341555, "acc_ci": [ -0.5253940455341444, 1.5761821366024553 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.0057862434257849316, "auroc_ci": [ -0.00016326111963524954, 0.0118510232345322 ], "acc_diff": 1.4227642276422703, "acc_ci": [ 0.06775067750677266, 2.777777777777779 ], "n": 1476 }, "atbench": { "auroc_diff": 0.007654275553919954, "auroc_ci": [ -0.0005845293412894525, 0.015683348892346016 ], "acc_diff": -1.200000000000001, "acc_ci": [ -3.0000000000000027, 0.5000000000000004 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.007663999999999893, "auroc_ci": [ -0.0004176276507143416, 0.015846713114484885 ], "acc_diff": -0.6000000000000005, "acc_ci": [ -2.400000000000002, 1.19999999999999 ], "n": 500 }, "atbench_claw": { "auroc_diff": 0.005522986221515702, "auroc_ci": [ -0.0016574485983901627, 0.014226281586627738 ], "acc_diff": 0.8000000000000007, "acc_ci": [ -1.0000000000000009, 2.8000000000000025 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": -0.00442998145204021, "auroc_ci": [ -0.010003026224938621, 0.0006491656054350765 ], "acc_diff": 1.6000000000000014, "acc_ci": [ 0.20000000000000018, 3.0000000000000027 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.03475082079213554, "auroc_ci": [ 0.025578268193739158, 0.04450371830182895 ], "acc_diff": 3.418230563002683, "acc_ci": [ 2.278820375335122, 4.6246648793565655 ], "n": 1492 }, "tracesafe": { "auroc_diff": -0.015704732510288144, "auroc_ci": [ -0.030325944480356318, -0.0014425933376769774 ], "acc_diff": 0.512820512820511, "acc_ci": [ -0.9401709401709368, 1.9658119658119588 ], "n": 1170 } }, "h52_w25_18k": { "rjudge": { "auroc_diff": 0.0030207948812599472, "auroc_ci": [ -0.0006662832788369043, 0.00647551673952085 ], "acc_diff": 0.0, "acc_ci": [ -1.2259194395796813, 1.2259194395796813 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.014332956551795117, "auroc_ci": [ 0.00971134735718637, 0.018917065309722023 ], "acc_diff": -1.8970189701897011, "acc_ci": [ -3.1165311653116534, -0.6775067750677488 ], "n": 1476 }, "atbench": { "auroc_diff": 0.005948214135708896, "auroc_ci": [ -0.003357810176421841, 0.014553822150768288 ], "acc_diff": -2.100000000000002, "acc_ci": [ -4.0000000000000036, -0.20000000000000018 ], "n": 1000 }, "atbench_codex": { "auroc_diff": -0.0025760000000000227, "auroc_ci": [ -0.012564109948060662, 0.006973715802110322 ], "acc_diff": 1.19999999999999, "acc_ci": [ -0.40000000000000036, 2.8000000000000025 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.05734134870164276, "auroc_ci": [ -0.07721304284567534, -0.039948984665650486 ], "acc_diff": -7.9999999999999964, "acc_ci": [ -10.999999999999998, -5.200000000000005 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": -0.002898118706942232, "auroc_ci": [ -0.00874739355784617, 0.0029525478993108903 ], "acc_diff": -4.399999999999993, "acc_ci": [ -6.605000000000006, -2.200000000000002 ], "n": 500 }, "agentdyn": { "auroc_diff": -0.011218283263431661, "auroc_ci": [ -0.016267818331435903, -0.006887531328320793 ], "acc_diff": -4.825737265415553, "acc_ci": [ -6.367292225201071, -3.2841823056300345 ], "n": 1492 }, "tracesafe": { "auroc_diff": -0.0016975308641975495, "auroc_ci": [ -0.019429578346706503, 0.01685272711243418 ], "acc_diff": 2.136752136752129, "acc_ci": [ 0.5982905982905962, 3.6752136752136844 ], "n": 1170 } } }, "E3_atbench_acc": { "ours": 72.5, "unified": 78.3 }, "sets": { "atbench_codex": [ 0.923, 81.6 ], "atbench": [ 0.839, 72.5 ], "rjudge": [ 0.978, 91.6 ], "assebench_strict": [ 0.88, 71.3 ], "atbench_claw_native": [ 0.928, 80.2 ], "agentdyn": [ 0.977, 51.2 ], "tracesafe": [ 0.559, 57.7 ] }, "X4": { "rjudge": [ 91.6, 88.6 ], "assebench_strict": [ 71.3, 62.5 ], "atbench": [ 72.5, 78.4 ], "atbench_codex": [ 81.6, 83.8 ], "tau": 0.72 }, "vs_unified_all_sets": { "rjudge": [ 0.021, [ 0.006, 0.037 ], 1.6 ], "assebench_strict": [ 0.049, [ 0.03, 0.068 ], -8.3 ], "atbench": [ -0.005, [ -0.027, 0.018 ], -5.8 ], "atbench_codex": [ 0.03, [ 0.006, 0.055 ], -1.6 ], "atbench_claw": [ -0.028, [ -0.067, 0.011 ], -13.0 ], "atbench_claw_native": [ 0.037, [ 0.016, 0.061 ], -8.6 ], "agentdyn": [ 0.302, [ 0.272, 0.332 ], 6.3 ], "tracesafe": [ -0.021, [ -0.078, 0.037 ], -4.3 ] } } }, "E1_supported": false, "E2_supported": false }