{ "sets": { "rjudge": { "n": 571, "missing": 0, "auroc": 0.9573151224314015, "acc": 90.01751313485113, "f1": 90.2229845626072, "published": 90.4, "reproduced": true }, "assebench_strict": { "n": 1476, "missing": 0, "auroc": 0.8310004016927144, "acc": 79.53929539295393, "f1": 79.45578231292517 }, "atbench": { "n": 1000, "missing": 0, "auroc": 0.8438823797656717, "acc": 78.3, "f1": 77.41935483870968, "published": 78.4, "reproduced": true }, "atbench_codex": { "n": 500, "missing": 0, "auroc": 0.893264, "acc": 83.2, "f1": 84.21052631578947 }, "atbench_claw": { "n": 500, "missing": 0, "auroc": 0.8878759273979862, "acc": 87.6, "f1": 89.8360655737705 }, "atbench_claw_native": { "n": 500, "missing": 0, "auroc": 0.8906581213566508, "acc": 88.8, "f1": 90.81967213114754 }, "agentdyn": { "n": 1486, "missing": 4864, "auroc": 0.6792305885848754, "acc": 44.751009421265145, "f1": 27.409372236958447 }, "tracesafe": { "n": 1170, "missing": 0, "auroc": 0.5799948559670782, "acc": 61.965811965811966, "f1": 75.50908090258667, "baa_0.5": 53.425925925925924 } }, "vs_guards": { "rjudge": { "agentdog_4b": { "auroc_diff": -0.009819121447028345, "auroc_ci": [ -0.029417630933077036, 0.01036861132294604 ], "acc_diff": 1.5761821366024442, "acc_ci": [ -1.5761821366024553, 4.728546409807355 ], "n": 571 }, "stepguard_4b": { "auroc_diff": 0.024861572535991416, "auroc_ci": [ 0.0038273730049030066, 0.046898997689975214 ], "acc_diff": 0.5253940455341444, "acc_ci": [ -1.7513134851138368, 2.977232924693518 ], "n": 571 }, "agentdog15_4b": { "auroc_diff": -0.012919896640826933, "auroc_ci": [ -0.027970378301906497, 0.001484768215849368 ], "acc_diff": -0.350262697022774, "acc_ci": [ -2.806479859894921, 2.1015761821365997 ], "n": 571 } }, "atbench": { "agentdog_4b": { "auroc_diff": 0.24133668812077258, "auroc_ci": [ 0.20753934766084922, 0.27539971488993115 ], "acc_diff": 18.100000000000005, "acc_ci": [ 14.497500000000002, 21.70000000000001 ], "n": 1000 }, "stepguard_4b": { "auroc_diff": 0.05641003076110751, "auroc_ci": [ 0.027528281897365746, 0.08525293445577453 ], "acc_diff": 5.300000000000004, "acc_ci": [ 2.6000000000000023, 8.200000000000006 ], "n": 1000 }, "agentdog15_4b": { "auroc_diff": 0.10157892787628642, "auroc_ci": [ 0.07303585823634727, 0.1287921679333249 ], "acc_diff": 10.649087221095332, "acc_ci": [ 7.505070993914808, 13.691683569979718 ], "n": 986 } }, "atbench_codex": { "agentdog_4b": { "auroc_diff": 0.06046399999999996, "auroc_ci": [ 0.026949525969834954, 0.09593771105659839 ], "acc_diff": 2.9999999999999916, "acc_ci": [ -0.40000000000000036, 6.599999999999994 ], "n": 500 }, "stepguard_4b": { "auroc_diff": 0.05075200000000002, "auroc_ci": [ 0.0193557043528508, 0.08398019999999978 ], "acc_diff": 4.999999999999993, "acc_ci": [ 1.5999999999999903, 8.599999999999996 ], "n": 500 }, "agentdog15_4b": { "auroc_diff": 0.018779571728208322, "auroc_ci": [ -0.012691299924244883, 0.05113527583531361 ], "acc_diff": 5.645161290322587, "acc_ci": [ 2.016129032258074, 9.2741935483871 ], "n": 496 } } }, "arms": { "h32_s0": { "rjudge": { "auroc_diff": 0.007874984619170688, "auroc_ci": [ -0.007952686688214841, 0.024295175703981976 ], "acc_diff": -2.4518388791593626, "acc_ci": [ -4.903677758318747, 0.0 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.03440171139779724, "auroc_ci": [ 0.014536256936952666, 0.05408362907439627 ], "acc_diff": -14.02439024390244, "acc_ci": [ -16.327913279132787, -11.788617886178866 ], "n": 1476 }, "atbench": { "auroc_diff": -0.012166437991767975, "auroc_ci": [ -0.03570004917010014, 0.01121509092667737 ], "acc_diff": -4.900000000000004, "acc_ci": [ -7.700000000000006, -2.100000000000002 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.03273599999999999, "auroc_ci": [ 0.007647562328205493, 0.05731726698312903 ], "acc_diff": -1.9999999999999907, "acc_ci": [ -4.999999999999993, 1.19999999999999 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.0884009009009008, "auroc_ci": [ -0.1353539010617551, -0.04307528012113371 ], "acc_diff": -17.400000000000006, "acc_ci": [ -21.799999999999997, -13.200000000000001 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.006673953365129881, "auroc_ci": [ -0.016476271766343024, 0.0302829514170535 ], "acc_diff": -12.2, "acc_ci": [ -15.800000000000002, -8.600000000000009 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.3023584611771697, "auroc_ci": [ 0.2713873845425281, 0.3321378377437629 ], "acc_diff": 24.29340511440108, "acc_ci": [ 22.139973082099594, 26.446837146702556 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.002572016460905324, "auroc_ci": [ -0.05531056109153116, 0.05193769749625696 ], "acc_diff": -14.95726495726496, "acc_ci": [ -17.863247863247867, -11.965811965811973 ], "n": 1170 } }, "h32_s1": { "rjudge": { "auroc_diff": 0.018838439768672388, "auroc_ci": [ 0.003325026556980132, 0.03512521575090488 ], "acc_diff": 1.4010507880910739, "acc_ci": [ -0.8800350262697026, 3.8528896672504365 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.033109755527945595, "auroc_ci": [ 0.012206340680426175, 0.05325250068786603 ], "acc_diff": -11.043360433604343, "acc_ci": [ -13.34688346883468, -8.738143631436314 ], "n": 1476 }, "atbench": { "auroc_diff": -0.01635658883719826, "auroc_ci": [ -0.040664058974714444, 0.006672728700192823 ], "acc_diff": -8.900000000000007, "acc_ci": [ -11.899999999999999, -5.900000000000006 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.013720000000000065, "auroc_ci": [ -0.014216963202614254, 0.03959793810315116 ], "acc_diff": -6.399999999999995, "acc_ci": [ -9.999999999999998, -2.9999999999999916 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.047810678325384104, "auroc_ci": [ -0.09032133512865875, -0.005748119866717939 ], "acc_diff": -16.200000000000003, "acc_ci": [ -20.599999999999994, -12.4 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.03139076576576583, "auroc_ci": [ 0.009191080144844165, 0.055169730282897005 ], "acc_diff": -13.600000000000001, "acc_ci": [ -17.400000000000006, -9.999999999999998 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.29804507962981597, "auroc_ci": [ 0.26761653461665763, 0.32717298082038915 ], "acc_diff": 12.314939434724092, "acc_ci": [ 10.296096904441454, 14.333781965006725 ], "n": 1486 }, "tracesafe": { "auroc_diff": 0.01783950617283947, "auroc_ci": [ -0.04135183514696254, 0.07797434311313615 ], "acc_diff": -5.384615384615388, "acc_ci": [ -7.948717948717943, -2.564102564102566 ], "n": 1170 } }, "h32_s2": { "rjudge": { "auroc_diff": 0.00816414421065581, "auroc_ci": [ -0.006731936641794512, 0.024024713236699097 ], "acc_diff": -0.3502626970227629, "acc_ci": [ -2.8021015761821366, 1.9264448336252185 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.03751342868085983, "auroc_ci": [ 0.016719410285598747, 0.058187898194103735 ], "acc_diff": -10.433604336043366, "acc_ci": [ -12.601626016260159, -8.265582655826552 ], "n": 1476 }, "atbench": { "auroc_diff": -0.020436735722486055, "auroc_ci": [ -0.045602033180343796, 0.003840008888065487 ], "acc_diff": -5.400000000000005, "acc_ci": [ -8.500000000000007, -2.500000000000002 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.023496000000000072, "auroc_ci": [ -0.0017409265232811554, 0.04835289481209381 ], "acc_diff": -2.7999999999999914, "acc_ci": [ -5.999999999999995, 0.40000000000000036 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.07381094329623727, "auroc_ci": [ -0.11887612070147412, -0.029410936924792776 ], "acc_diff": -14.000000000000002, "acc_ci": [ -18.200000000000006, -9.999999999999998 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.029229597244303074, "auroc_ci": [ 0.005686636857774261, 0.05460499125812438 ], "acc_diff": -8.999999999999996, "acc_ci": [ -12.4, -5.999999999999995 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.2921285579406957, "auroc_ci": [ 0.2619928308751839, 0.32060527121123084 ], "acc_diff": 17.29475100942126, "acc_ci": [ 15.208613728129205, 19.17900403768506 ], "n": 1486 }, "tracesafe": { "auroc_diff": 0.011064814814814805, "auroc_ci": [ -0.04049923614432227, 0.06602622144004643 ], "acc_diff": -14.444444444444448, "acc_ci": [ -17.179487179487186, -11.623931623931627 ], "n": 1170 } }, "teach_s0": { "rjudge": { "auroc_diff": 0.026664205733973212, "auroc_ci": [ 0.011744003450816496, 0.042855352720441636 ], "acc_diff": 3.1523642732049106, "acc_ci": [ 1.0507880910682998, 5.604203152364273 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.038058983437181726, "auroc_ci": [ 0.0180024283043435, 0.05757223622524813 ], "acc_diff": -9.55284552845529, "acc_ci": [ -11.720867208672093, -7.3848238482384865 ], "n": 1476 }, "atbench": { "auroc_diff": -0.04474161069798532, "auroc_ci": [ -0.0712638574041776, -0.01988803078884324 ], "acc_diff": -7.300000000000006, "acc_ci": [ -10.20000000000001, -4.397500000000018 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.017967999999999984, "auroc_ci": [ -0.009788141231142172, 0.043838586328408806 ], "acc_diff": -0.6000000000000005, "acc_ci": [ -3.599999999999992, 2.599999999999991 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.05999105723370435, "auroc_ci": [ -0.1037707087780301, -0.016298044578733725 ], "acc_diff": -12.8, "acc_ci": [ -16.800000000000004, -8.999999999999996 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.030388844727079922, "auroc_ci": [ 0.007636752082114467, 0.0544938358970897 ], "acc_diff": -7.599999999999996, "acc_ci": [ -10.599999999999998, -4.600000000000004 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.21798871811093057, "auroc_ci": [ 0.18676343194155612, 0.24792080162965838 ], "acc_diff": 0.538358008075368, "acc_ci": [ -1.211305518169581, 2.2880215343203227 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.05831275720164619, "auroc_ci": [ -0.11687461041779995, -0.0007434666158338158 ], "acc_diff": -8.461538461538465, "acc_ci": [ -10.940170940170946, -5.897435897435899 ], "n": 1170 } }, "teach_s1": { "rjudge": { "auroc_diff": 0.024110988064476424, "auroc_ci": [ 0.00915685507969467, 0.04010427955372052 ], "acc_diff": 3.5026269702276736, "acc_ci": [ 1.0507880910682998, 5.954465849387036 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.03995441254775933, "auroc_ci": [ 0.019684339264772234, 0.06042728154015862 ], "acc_diff": -4.40379403794039, "acc_ci": [ -6.57350948509485, -2.2357723577235755 ], "n": 1476 }, "atbench": { "auroc_diff": -0.03151913468884893, "auroc_ci": [ -0.055698101229814195, -0.008391155960339345 ], "acc_diff": -11.299999999999999, "acc_ci": [ -14.400000000000002, -8.000000000000007 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.013072000000000084, "auroc_ci": [ -0.013858453292753726, 0.03908661769486731 ], "acc_diff": -3.7999999999999923, "acc_ci": [ -7.199999999999996, -0.40000000000000036 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.06133247217806015, "auroc_ci": [ -0.10408844679606112, -0.016616533974221102 ], "acc_diff": -13.4, "acc_ci": [ -17.400000000000006, -9.595000000000026 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.027258876523582276, "auroc_ci": [ 0.004533656107525028, 0.05203776677225913 ], "acc_diff": -9.199999999999998, "acc_ci": [ -12.4, -6.000000000000005 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.2809736739946226, "auroc_ci": [ 0.2516919730235246, 0.309084246694381 ], "acc_diff": -4.441453566621806, "acc_ci": [ -6.1911170928667545, -2.759084791386268 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.0335185185185185, "auroc_ci": [ -0.09381174802031764, 0.027280285613804252 ], "acc_diff": 2.3931623931623958, "acc_ci": [ 0.0, 4.700854700854706 ], "n": 1170 } }, "teach_s2": { "rjudge": { "auroc_diff": 0.02253599114064231, "auroc_ci": [ 0.008199521078188624, 0.03761176078154388 ], "acc_diff": 2.101576182136611, "acc_ci": [ -0.17513134851138146, 4.378283712784581 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.0416022868459649, "auroc_ci": [ 0.022631654817631023, 0.06038439355360588 ], "acc_diff": -6.165311653116534, "acc_ci": [ -8.19783197831978, -4.132791327913276 ], "n": 1476 }, "atbench": { "auroc_diff": -0.04098747554911997, "auroc_ci": [ -0.0681189814160153, -0.017208530982927295 ], "acc_diff": -8.000000000000007, "acc_ci": [ -11.099999999999998, -5.197500000000018 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.032056000000000084, "auroc_ci": [ 0.005959563984709096, 0.056824524737431235 ], "acc_diff": 0.20000000000000018, "acc_ci": [ -2.8000000000000025, 3.400000000000003 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.0756243375728669, "auroc_ci": [ -0.12093549248642425, -0.030848845982364478 ], "acc_diff": -15.000000000000002, "acc_ci": [ -19.399999999999995, -10.799999999999999 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.015244104398515979, "auroc_ci": [ -0.0072457614942529, 0.03926520807191212 ], "acc_diff": -11.799999999999999, "acc_ci": [ -15.600000000000003, -8.399999999999997 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.25762869453111137, "auroc_ci": [ 0.22860609110139235, 0.28681301151543037 ], "acc_diff": -4.845222072678334, "acc_ci": [ -6.393001345895022, -3.2974427994616473 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.021430041152263435, "auroc_ci": [ -0.08340559165120655, 0.040954313259502 ], "acc_diff": 2.478632478632481, "acc_ci": [ 0.2564102564102555, 4.702991452991447 ], "n": 1170 } }, "w25": { "rjudge": { "auroc_diff": 0.027297895902547054, "auroc_ci": [ 0.012621611780857056, 0.043657088143969285 ], "acc_diff": 0.8756567425569184, "acc_ci": [ -1.2259194395796924, 3.1523642732048995 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.04647491288873118, "auroc_ci": [ 0.026748617806584617, 0.06603242654899324 ], "acc_diff": -13.482384823848248, "acc_ci": [ -15.650406504065039, -11.314363143631434 ], "n": 1476 }, "atbench": { "auroc_diff": -0.021236764523523077, "auroc_ci": [ -0.045535106987388815, 0.0012045599563388937 ], "acc_diff": -6.400000000000006, "acc_ci": [ -9.200000000000008, -3.400000000000003 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.024720000000000075, "auroc_ci": [ -0.00038560806857672325, 0.04928488078081273 ], "acc_diff": -0.40000000000000036, "acc_ci": [ -3.399999999999992, 2.7999999999999914 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.04604696608373082, "auroc_ci": [ -0.08747025206226479, -0.004249828857385338 ], "acc_diff": -11.6, "acc_ci": [ -15.600000000000003, -7.9999999999999964 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.023930180180180227, "auroc_ci": [ 0.0010412177613762315, 0.047593642037737006 ], "acc_diff": -7.599999999999996, "acc_ci": [ -10.600000000000009, -4.600000000000004 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.28149367610338694, "auroc_ci": [ 0.25211135864905704, 0.3106064104066566 ], "acc_diff": 10.026917900403765, "acc_ci": [ 8.07537012113056, 11.845558546433368 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.05531893004115229, "auroc_ci": [ -0.11433787270642207, 0.002988992890915766 ], "acc_diff": -15.64102564102564, "acc_ci": [ -18.46367521367522, -12.820512820512825 ], "n": 1170 } }, "w25_s1": { "rjudge": { "auroc_diff": 0.021428571428571463, "auroc_ci": [ 0.007203129469589909, 0.03743749909351522 ], "acc_diff": 1.5761821366024553, "acc_ci": [ -0.8756567425569184, 4.028021015761829 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.040473810569189306, "auroc_ci": [ 0.01948081899076893, 0.06048275263461059 ], "acc_diff": -6.77506775067751, "acc_ci": [ -9.010840108401085, -4.607046070460696 ], "n": 1476 }, "atbench": { "auroc_diff": -0.020466736802524976, "auroc_ci": [ -0.04476283331154423, 0.002082946088219829 ], "acc_diff": -10.399999999999999, "acc_ci": [ -13.5025, -7.199999999999996 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.011152000000000162, "auroc_ci": [ -0.018649670254575257, 0.038970926940538976 ], "acc_diff": -5.199999999999994, "acc_ci": [ -8.799999999999997, -1.7999999999999905 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.05551139374668779, "auroc_ci": [ -0.09812222191422454, -0.012014932463994986 ], "acc_diff": -13.8, "acc_ci": [ -17.800000000000004, -9.799999999999997 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.037037957074721684, "auroc_ci": [ 0.015310338720601623, 0.06137738375130607 ], "acc_diff": -9.999999999999998, "acc_ci": [ -13.200000000000001, -6.799999999999995 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.2797227933458901, "auroc_ci": [ 0.24973473654440007, 0.3081108558164418 ], "acc_diff": 0.8748317631224745, "acc_ci": [ -0.8748317631224745, 2.4899057873485897 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.017227366255144094, "auroc_ci": [ -0.07700874705521323, 0.04091141798973637 ], "acc_diff": -2.7350427350427364, "acc_ci": [ -5.301282051282055, -0.2564102564102555 ], "n": 1170 } }, "w25_s2": { "rjudge": { "auroc_diff": 0.015183954718838488, "auroc_ci": [ 0.00047657321685243287, 0.03083053817962469 ], "acc_diff": 2.8021015761821366, "acc_ci": [ 0.5253940455341444, 5.078809106830128 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.05083000924827408, "auroc_ci": [ 0.03150054163328806, 0.06927791401082924 ], "acc_diff": -4.20054200542006, "acc_ci": [ -6.165311653116534, -2.2357723577235755 ], "n": 1476 }, "atbench": { "auroc_diff": -0.008218295858651081, "auroc_ci": [ -0.03286021643952745, 0.01577692383379831 ], "acc_diff": -5.300000000000004, "acc_ci": [ -8.300000000000008, -2.500000000000002 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.018208000000000002, "auroc_ci": [ -0.006853488877162081, 0.04265247811314437 ], "acc_diff": -2.199999999999991, "acc_ci": [ -5.199999999999994, 1.0000000000000009 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.07412559618441972, "auroc_ci": [ -0.12038361289401471, -0.028799876174581795 ], "acc_diff": -17.800000000000004, "acc_ci": [ -22.199999999999996, -13.600000000000001 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.024070945945945832, "auroc_ci": [ 0.0010107289280552939, 0.04834301867031254 ], "acc_diff": -12.2, "acc_ci": [ -15.805000000000001, -8.799999999999997 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.2758191830455349, "auroc_ci": [ 0.24683172399746614, 0.3043445292061223 ], "acc_diff": 4.777927321668907, "acc_ci": [ 2.960969044414541, 6.594885598923284 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.014346707818930104, "auroc_ci": [ -0.07047452223857836, 0.04304651186572272 ], "acc_diff": 2.991452991452992, "acc_ci": [ 0.5982905982905962, 5.555555555555558 ], "n": 1170 } }, "w25u": { "rjudge": { "auroc_diff": 0.008994708994709, "auroc_ci": [ -0.0067606212609408415, 0.02496392057762967 ], "acc_diff": 1.0507880910682998, "acc_ci": [ -1.0507880910682998, 3.1523642732049106 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.04319878184348935, "auroc_ci": [ 0.02373424861880531, 0.06272749432351682 ], "acc_diff": -9.688346883468835, "acc_ci": [ -11.788617886178853, -7.72357723577236 ], "n": 1476 }, "atbench": { "auroc_diff": -0.012290442455928585, "auroc_ci": [ -0.03481184095060911, 0.008839146420084905 ], "acc_diff": -4.600000000000004, "acc_ci": [ -7.400000000000007, -1.9000000000000017 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.02254400000000012, "auroc_ci": [ -0.001617301510464844, 0.046240166942443615 ], "acc_diff": -1.0000000000000009, "acc_ci": [ -3.7999999999999923, 1.999999999999991 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.033601616322204486, "auroc_ci": [ -0.07546856229137383, 0.007421089313318868 ], "acc_diff": -13.8, "acc_ci": [ -17.800000000000004, -9.999999999999998 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.04160042395336505, "auroc_ci": [ 0.01904934817777225, 0.06656498215558034 ], "acc_diff": -10.199999999999998, "acc_ci": [ -13.405, -6.999999999999995 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.26762136178248097, "auroc_ci": [ 0.2382663703585987, 0.29672721837869803 ], "acc_diff": 2.893674293405113, "acc_ci": [ 1.3458950201884257, 4.441453566621806 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.00508230452674896, "auroc_ci": [ -0.0590298745773972, 0.04828723445989927 ], "acc_diff": -4.7863247863247915, "acc_ci": [ -7.179487179487188, -2.3076923076923106 ], "n": 1170 } }, "w25_18k": { "rjudge": { "auroc_diff": 0.01794635166728198, "auroc_ci": [ 0.00312814569761907, 0.033386208316901166 ], "acc_diff": 1.5761821366024553, "acc_ci": [ -0.7005253940455369, 3.857267950963208 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.03465206871747917, "auroc_ci": [ 0.014353688748294865, 0.05431376352931247 ], "acc_diff": -6.368563685636863, "acc_ci": [ -8.536585365853655, -4.268292682926833 ], "n": 1476 }, "atbench": { "auroc_diff": -0.010584381037717527, "auroc_ci": [ -0.0339783419882701, 0.012431406432390258 ], "acc_diff": -3.7000000000000033, "acc_ci": [ -6.400000000000006, -1.0000000000000009 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.032784000000000035, "auroc_ci": [ 0.007951096236559086, 0.05746341749322339 ], "acc_diff": -2.7999999999999914, "acc_ci": [ -5.999999999999995, 0.40000000000000036 ], "n": 500 }, "atbench_claw": { "auroc_diff": 0.029262718600953974, "auroc_ci": [ -0.0013578393814041437, 0.06156798416228268 ], "acc_diff": -5.000000000000004, "acc_ci": [ -8.399999999999997, -1.8000000000000016 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.04006856120826707, "auroc_ci": [ 0.018364333178678822, 0.06341901469382968 ], "acc_diff": -4.200000000000004, "acc_ci": [ -6.800000000000006, -1.6000000000000014 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.3126147239675441, "auroc_ci": [ 0.28160570135852625, 0.34312658732202345 ], "acc_diff": 11.37281292059219, "acc_ci": [ 9.42126514131898, 13.25706594885599 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.019089506172839554, "auroc_ci": [ -0.08283397000943271, 0.0450631162242672 ], "acc_diff": -6.41025641025641, "acc_ci": [ -8.803418803418806, -3.8461538461538436 ], "n": 1170 } }, "gold": { "rjudge": { "auroc_diff": -0.037289282638119925, "auroc_ci": [ -0.05610189636446871, -0.020278410559548307 ], "acc_diff": -5.2539404553415, "acc_ci": [ -8.056042031523647, -2.626970227670755 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.01216755257036628, "auroc_ci": [ -0.011895602495230992, 0.03578209939713676 ], "acc_diff": -15.108401084010847, "acc_ci": [ -17.479674796747968, -12.60162601626017 ], "n": 1476 }, "atbench": { "auroc_diff": 0.00901832465968766, "auroc_ci": [ -0.018136955554487723, 0.03545533682789022 ], "acc_diff": -5.800000000000005, "acc_ci": [ -9.100000000000009, -2.497500000000016 ], "n": 1000 }, "atbench_codex": { "auroc_diff": -0.008975999999999984, "auroc_ci": [ -0.03999844521269604, 0.02135457202701558 ], "acc_diff": -15.199999999999992, "acc_ci": [ -19.599999999999994, -10.999999999999998 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.05898913619501844, "auroc_ci": [ -0.10452264917653872, -0.01267085371400443 ], "acc_diff": -20.799999999999997, "acc_ci": [ -25.4, -16.39999999999999 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.03184618441971376, "auroc_ci": [ 0.005354780299134074, 0.05881589469400627 ], "acc_diff": -15.600000000000003, "acc_ci": [ -19.600000000000005, -11.799999999999999 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.3043785615351805, "auroc_ci": [ 0.27341592250735086, 0.33398096554827644 ], "acc_diff": 44.34724091520861, "acc_ci": [ 41.790040376850605, 46.90444145356661 ], "n": 1486 }, "tracesafe": { "auroc_diff": 0.05336419753086419, "auroc_ci": [ -0.0065034560355295866, 0.11128625064757817 ], "acc_diff": -29.572649572649574, "acc_ci": [ -32.9059829059829, -26.239316239316242 ], "n": 1170 } }, "3b_mix": { "rjudge": { "auroc_diff": 0.009130060292851017, "auroc_ci": [ -0.005417968121949132, 0.024178117695591493 ], "acc_diff": -1.0507880910682998, "acc_ci": [ -3.502626970227663, 1.4010507880910739 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.03561706540117893, "auroc_ci": [ 0.014599626301613725, 0.05611238535852722 ], "acc_diff": -15.311653116531165, "acc_ci": [ -17.75067750677507, -12.87262872628726 ], "n": 1476 }, "atbench": { "auroc_diff": -0.024686888727994405, "auroc_ci": [ -0.049083078753780846, 0.00031826997567228205 ], "acc_diff": -8.300000000000008, "acc_ci": [ -11.299999999999999, -5.397500000000019 ], "n": 1000 }, "atbench_codex": { "auroc_diff": -0.007399999999999962, "auroc_ci": [ -0.03835454567452978, 0.02069721745066948 ], "acc_diff": -4.999999999999993, "acc_ci": [ -8.199999999999996, -1.6000000000000012 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.040333532061473165, "auroc_ci": [ -0.08366551061764192, 0.0015882133881504662 ], "acc_diff": -17.600000000000005, "acc_ci": [ -21.999999999999996, -13.200000000000001 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.034487612612612684, "auroc_ci": [ 0.010959511729462357, 0.05876145691266628 ], "acc_diff": -12.6, "acc_ci": [ -16.400000000000002, -9.199999999999998 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.30546888853742815, "auroc_ci": [ 0.27421938528026124, 0.3357120191047023 ], "acc_diff": 13.324360699865412, "acc_ci": [ 11.50740242261104, 15.210296096904438 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.04898662551440336, "auroc_ci": [ -0.10335903707603565, 0.005058694408387157 ], "acc_diff": -14.188034188034187, "acc_ci": [ -17.00854700854701, -11.450854700854705 ], "n": 1170 } }, "3b_teach": { "rjudge": { "auroc_diff": 0.016992740248554128, "auroc_ci": [ 0.0025182511123934105, 0.032441137542234814 ], "acc_diff": 1.4010507880910739, "acc_ci": [ -0.7005253940455369, 3.507005253940434 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.040397208702719456, "auroc_ci": [ 0.0203002812293044, 0.05971556009090536 ], "acc_diff": -11.44986449864499, "acc_ci": [ -13.617886178861793, -9.281842818428188 ], "n": 1476 }, "atbench": { "auroc_diff": -0.05587801160841799, "auroc_ci": [ -0.08104927705274337, -0.03068295305432122 ], "acc_diff": -10.799999999999999, "acc_ci": [ -13.900000000000002, -7.800000000000007 ], "n": 1000 }, "atbench_codex": { "auroc_diff": -0.005856000000000083, "auroc_ci": [ -0.033976692102995795, 0.022046371009919456 ], "acc_diff": -4.799999999999994, "acc_ci": [ -8.199999999999996, -1.4000000000000012 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.03600291467938521, "auroc_ci": [ -0.07600106832714432, 0.004992177833657996 ], "acc_diff": -11.399999999999999, "acc_ci": [ -15.600000000000003, -7.599999999999996 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.016925013248542675, "auroc_ci": [ -0.007029184936195254, 0.042088964622115047 ], "acc_diff": -5.400000000000005, "acc_ci": [ -8.205000000000005, -2.8000000000000025 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.27634877044662665, "auroc_ci": [ 0.24634185587341254, 0.3064234033517196 ], "acc_diff": -1.8842530282637937, "acc_ci": [ -3.6339165545087484, -0.0672947510094224 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.0523508230452675, "auroc_ci": [ -0.11615654413509452, 0.00825182301659372 ], "acc_diff": -6.75213675213675, "acc_ci": [ -9.487179487179487, -3.9316239316239288 ], "n": 1170 } }, "ens_w25": { "rjudge": { "auroc_diff": 0.02201304294327555, "auroc_ci": [ 0.007454665057679738, 0.03791683641457943 ], "acc_diff": 2.2767075306479923, "acc_ci": [ 0.0, 4.728546409807355 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.04950255495249756, "auroc_ci": [ 0.030079140399036464, 0.06825665201986864 ], "acc_diff": -7.92682926829269, "acc_ci": [ -10.162601626016254, -5.7588075880758876 ], "n": 1476 }, "atbench": { "auroc_diff": -0.00970834950058208, "auroc_ci": [ -0.03346510544943757, 0.013041593711894398 ], "acc_diff": -7.000000000000006, "acc_ci": [ -9.90000000000001, -4.200000000000004 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.026912000000000047, "auroc_ci": [ 0.0015388753759192797, 0.051465866128847744 ], "acc_diff": -1.8000000000000016, "acc_ci": [ -4.799999999999994, 1.200000000000001 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.05609929782723877, "auroc_ci": [ -0.09980937130359255, -0.013004912028252665 ], "acc_diff": -13.4, "acc_ci": [ -17.400000000000006, -9.399999999999997 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.0301652755696874, "auroc_ci": [ 0.00826929534522002, 0.053885153659750266 ], "acc_diff": -9.999999999999998, "acc_ci": [ -13.4, -6.799999999999995 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.28336760075340406, "auroc_ci": [ 0.25365637678009606, 0.3123266857844479 ], "acc_diff": 5.383580080753703, "acc_ci": [ 3.7668236877523524, 7.133243606998652 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.030236625514403315, "auroc_ci": [ -0.08858884469752795, 0.026978939993188088 ], "acc_diff": -5.128205128205132, "acc_ci": [ -7.606837606837602, -2.564102564102555 ], "n": 1170 } }, "ens5": { "rjudge": { "auroc_diff": 0.020967146548541926, "auroc_ci": [ 0.006367251053043199, 0.03658942749894822 ], "acc_diff": 1.5761821366024553, "acc_ci": [ -0.5253940455341555, 3.677758318739055 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.048985025269274285, "auroc_ci": [ 0.02968558198250373, 0.06764499155631172 ], "acc_diff": -8.265582655826565, "acc_ci": [ -10.367547425474262, -6.165311653116534 ], "n": 1476 }, "atbench": { "auroc_diff": -0.004636166902008632, "auroc_ci": [ -0.027443739316762052, 0.017501228078596984 ], "acc_diff": -5.800000000000005, "acc_ci": [ -8.600000000000009, -3.0000000000000027 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.030208000000000013, "auroc_ci": [ 0.006398239792704952, 0.05452742921964889 ], "acc_diff": -1.6000000000000014, "acc_ci": [ -4.5999999999999925, 1.4000000000000012 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.028078630100688784, "auroc_ci": [ -0.06718745088943334, 0.011142848057641068 ], "acc_diff": -13.0, "acc_ci": [ -17.000000000000004, -9.199999999999998 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.03717044250132484, "auroc_ci": [ 0.015742992350449105, 0.06097563296146095 ], "acc_diff": -8.599999999999996, "acc_ci": [ -11.799999999999999, -5.595000000000022 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.30230813839245063, "auroc_ci": [ 0.2718789639962889, 0.3317150874352668 ], "acc_diff": 6.3257065948855935, "acc_ci": [ 4.7089502018842495, 8.008075370121132 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.020787037037037104, "auroc_ci": [ -0.07810256011557502, 0.03735814266191144 ], "acc_diff": -4.2735042735042805, "acc_ci": [ -6.75213675213675, -1.7948717948717996 ], "n": 1170 } }, "3b_w25": { "rjudge": { "auroc_diff": -0.0001538082933432161, "auroc_ci": [ -0.015089005392104194, 0.014787800188764985 ], "acc_diff": -1.7513134851138368, "acc_ci": [ -4.028021015761829, 0.5253940455341555 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.03768624996496861, "auroc_ci": [ 0.01664732173408001, 0.05790802715795375 ], "acc_diff": -13.27913279132792, "acc_ci": [ -15.582655826558266, -11.041666666666673 ], "n": 1476 }, "atbench": { "auroc_diff": -0.03211115600161629, "auroc_ci": [ -0.057444702762109896, -0.00788665354918257 ], "acc_diff": -9.300000000000008, "acc_ci": [ -12.4, -6.400000000000006 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.016959999999999975, "auroc_ci": [ -0.010081487959799804, 0.04343733246669551 ], "acc_diff": -2.9999999999999916, "acc_ci": [ -5.999999999999995, 0.20000000000000018 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.05013745363010069, "auroc_ci": [ -0.09312810911831905, -0.00886626948528849 ], "acc_diff": -14.400000000000002, "acc_ci": [ -18.600000000000005, -10.399999999999999 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.012933889772125173, "auroc_ci": [ -0.010202823134244223, 0.035816394946592514 ], "acc_diff": -13.4, "acc_ci": [ -17.000000000000004, -9.999999999999998 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.28702918242249087, "auroc_ci": [ 0.2568353566168098, 0.3170649500262892 ], "acc_diff": 4.71063257065949, "acc_ci": [ 2.8919919246298775, 6.596567967698515 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.020751028806584415, "auroc_ci": [ -0.07973499767017543, 0.03798546250600538 ], "acc_diff": -9.145299145299147, "acc_ci": [ -11.882478632478634, -6.239316239316239 ], "n": 1170 } }, "3b_w25_s1": { "rjudge": { "auroc_diff": 0.012753783684016362, "auroc_ci": [ -0.0022454290142446347, 0.028200462410647744 ], "acc_diff": -0.3502626970227629, "acc_ci": [ -2.8021015761821366, 2.1015761821365997 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.02844731753341989, "auroc_ci": [ 0.007618264488708323, 0.04923354143870089 ], "acc_diff": -12.059620596205967, "acc_ci": [ -14.29539295392953, -9.823848238482391 ], "n": 1476 }, "atbench": { "auroc_diff": -0.0327471788984407, "auroc_ci": [ -0.05761304311407259, -0.009461222572978614 ], "acc_diff": -9.599999999999998, "acc_ci": [ -12.7, -6.797500000000009 ], "n": 1000 }, "atbench_codex": { "auroc_diff": -0.0041119999999998935, "auroc_ci": [ -0.0351643216308248, 0.024437785359221992 ], "acc_diff": -5.399999999999993, "acc_ci": [ -8.604999999999995, -2.0000000000000018 ], "n": 500 }, "atbench_claw": { "auroc_diff": 0.009381624271330158, "auroc_ci": [ -0.025415928547894248, 0.044345837307283746 ], "acc_diff": -10.799999999999999, "acc_ci": [ -14.400000000000002, -7.199999999999996 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.04804252782193952, "auroc_ci": [ 0.0250733041553122, 0.07301151646486885 ], "acc_diff": -10.999999999999998, "acc_ci": [ -14.405000000000001, -7.599999999999996 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.25969192870459556, "auroc_ci": [ 0.2292592215015387, 0.28952765482590154 ], "acc_diff": 0.0, "acc_ci": [ -1.7496635262449491, 1.682368775235532 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.01669238683127572, "auroc_ci": [ -0.0804397493653384, 0.04331673556690563 ], "acc_diff": 0.9401709401709368, "acc_ci": [ -1.5384615384615332, 3.4188034188034178 ], "n": 1170 } }, "3b_w25_s2": { "rjudge": { "auroc_diff": 0.021145564168820075, "auroc_ci": [ 0.006700309964832024, 0.03752507076019452 ], "acc_diff": 1.0507880910682998, "acc_ci": [ -1.2259194395796924, 3.502626970227663 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.022828290377124194, "auroc_ci": [ 0.0031238371243867027, 0.042469731428552086 ], "acc_diff": -7.113821138211385, "acc_ci": [ -9.214092140921403, -4.945799457994582 ], "n": 1476 }, "atbench": { "auroc_diff": -0.04667368025248908, "auroc_ci": [ -0.07252411100497831, -0.02015684935744692 ], "acc_diff": -11.7, "acc_ci": [ -15.000000000000002, -8.599999999999996 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.007056000000000062, "auroc_ci": [ -0.019189635999327423, 0.03317416873704169 ], "acc_diff": -3.1999999999999917, "acc_ci": [ -6.199999999999994, 0.0 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.027441043985161673, "auroc_ci": [ -0.06808204727298119, 0.012193757699612196 ], "acc_diff": -15.200000000000003, "acc_ci": [ -19.599999999999994, -11.2 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.04169978802331742, "auroc_ci": [ 0.01895329629011267, 0.06555202835571576 ], "acc_diff": -11.6, "acc_ci": [ -15.200000000000003, -8.199999999999996 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.2955433183323507, "auroc_ci": [ 0.26399779503821413, 0.32420600526109844 ], "acc_diff": 2.0188425302826385, "acc_ci": [ 0.40376850605652326, 3.6339165545087484 ], "n": 1486 }, "tracesafe": { "auroc_diff": 0.02695987654320975, "auroc_ci": [ -0.03407387644017941, 0.08863156776589111 ], "acc_diff": 4.61538461538461, "acc_ci": [ 2.051282051282055, 7.094017094017091 ], "n": 1170 } }, "ens_3b": { "rjudge": { "auroc_diff": 0.014507198228128404, "auroc_ci": [ 0.0004782972336095232, 0.030349832756571066 ], "acc_diff": 1.0507880910682998, "acc_ci": [ -1.2259194395796924, 3.3274956217162814 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.034791259913869776, "auroc_ci": [ 0.01425403115026993, 0.05458960308483688 ], "acc_diff": -10.70460704607047, "acc_ci": [ -12.87262872628726, -8.468834688346893 ], "n": 1476 }, "atbench": { "auroc_diff": -0.02957306463032683, "auroc_ci": [ -0.05351164623209621, -0.005439255831255008 ], "acc_diff": -9.699999999999998, "acc_ci": [ -12.8, -6.800000000000006 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.0154080000000002, "auroc_ci": [ -0.01057826424070705, 0.04093074759841898 ], "acc_diff": -3.399999999999992, "acc_ci": [ -6.599999999999994, -0.19500000000001663 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.01888745363010058, "auroc_ci": [ -0.05865248862753025, 0.01992588846920391 ], "acc_diff": -14.400000000000002, "acc_ci": [ -18.599999999999994, -10.599999999999998 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.03986983306836245, "auroc_ci": [ 0.01788477243650026, 0.0637672544746054 ], "acc_diff": -12.0, "acc_ci": [ -15.600000000000003, -8.599999999999996 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.287728908762395, "auroc_ci": [ 0.2569482310615392, 0.31657538489873904 ], "acc_diff": 2.0861372812920607, "acc_ci": [ 0.4037685060565288, 3.7685060565275874 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.0014660493827161059, "auroc_ci": [ -0.06063631865899355, 0.056848360530981315 ], "acc_diff": -0.512820512820511, "acc_ci": [ -2.991452991452992, 2.0512820512820444 ], "n": 1170 } }, "3b_self": { "rjudge": { "auroc_diff": 0.015485418973791076, "auroc_ci": [ 0.0010110226351101649, 0.03146617492233492 ], "acc_diff": -1.4010507880910628, "acc_ci": [ -4.0280210157618175, 1.2259194395796813 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.021624146402981936, "auroc_ci": [ 0.0005453722566786445, 0.04222668857474019 ], "acc_diff": -12.804878048780488, "acc_ci": [ -15.176151761517609, -10.433604336043356 ], "n": 1476 }, "atbench": { "auroc_diff": -0.03696333067990454, "auroc_ci": [ -0.06175217559617585, -0.01319974826585274 ], "acc_diff": -9.999999999999998, "acc_ci": [ -13.0, -7.100000000000007 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.010040000000000049, "auroc_ci": [ -0.01861895740680239, 0.038350778012817964 ], "acc_diff": -3.599999999999992, "acc_ci": [ -6.999999999999995, -0.20000000000000018 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.09628378378378377, "auroc_ci": [ -0.14448567028229925, -0.04947067553895357 ], "acc_diff": -24.0, "acc_ci": [ -28.800000000000004, -19.399999999999995 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": -0.00780007949125594, "auroc_ci": [ -0.0314053141176175, 0.014285791946421558 ], "acc_diff": -20.399999999999995, "acc_ci": [ -24.6, -16.400000000000002 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.29179067638615297, "auroc_ci": [ 0.2599722617447743, 0.32133444731801747 ], "acc_diff": 6.796769851951545, "acc_ci": [ 4.845222072678334, 8.613728129205922 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.017026748971193495, "auroc_ci": [ -0.0769450072205232, 0.04312062155522243 ], "acc_diff": -8.37606837606838, "acc_ci": [ -11.111111111111116, -5.726495726495728 ], "n": 1170 } }, "3b_self_s1": { "rjudge": { "auroc_diff": 0.017128091546696145, "auroc_ci": [ 0.0030576391508850137, 0.032383092001418035 ], "acc_diff": -2.276707530647981, "acc_ci": [ -4.903677758318736, 0.17513134851139256 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.029203994507085662, "auroc_ci": [ 0.008684091523125268, 0.04879057133954991 ], "acc_diff": -13.821138211382122, "acc_ci": [ -16.124661246612458, -11.517615176151763 ], "n": 1476 }, "atbench": { "auroc_diff": -0.018476665159945838, "auroc_ci": [ -0.04230980813574747, 0.004282228001457482 ], "acc_diff": -8.600000000000009, "acc_ci": [ -11.50000000000001, -5.700000000000005 ], "n": 1000 }, "atbench_codex": { "auroc_diff": -0.006735999999999964, "auroc_ci": [ -0.03881613235338323, 0.02291633204942781 ], "acc_diff": -6.199999999999994, "acc_ci": [ -9.605000000000008, -2.800000000000002 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.020543521462639047, "auroc_ci": [ -0.063630458433249, 0.020243752869270513 ], "acc_diff": -17.400000000000006, "acc_ci": [ -21.605000000000008, -13.200000000000001 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.06025602808691044, "auroc_ci": [ 0.03585688066715767, 0.08753958427271824 ], "acc_diff": -13.8, "acc_ci": [ -17.400000000000006, -10.199999999999998 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.2991953147091103, "auroc_ci": [ 0.26848260861523987, 0.3278932550243101 ], "acc_diff": 9.555854643337813, "acc_ci": [ 7.804508748317624, 11.305518169582768 ], "n": 1486 }, "tracesafe": { "auroc_diff": -0.005612139917695491, "auroc_ci": [ -0.06695983574010736, 0.054552536323508594 ], "acc_diff": -12.564102564102564, "acc_ci": [ -15.470085470085465, -9.829059829059828 ], "n": 1170 } }, "3b_self_s2": { "rjudge": { "auroc_diff": 0.01711578688322868, "auroc_ci": [ 0.0029853125435418183, 0.03292415983176225 ], "acc_diff": -0.5253940455341444, "acc_ci": [ -2.977232924693518, 1.9264448336252182 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.039857258961017106, "auroc_ci": [ 0.01945249695118748, 0.05975702002922339 ], "acc_diff": -13.008130081300816, "acc_ci": [ -15.243902439024382, -10.704607046070457 ], "n": 1476 }, "atbench": { "auroc_diff": -0.03744934817653445, "auroc_ci": [ -0.06208566055056112, -0.01261420272462027 ], "acc_diff": -8.600000000000009, "acc_ci": [ -11.7, -5.700000000000005 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.0011760000000000659, "auroc_ci": [ -0.028211524855954493, 0.03049927508155185 ], "acc_diff": -5.599999999999994, "acc_ci": [ -8.999999999999996, -2.399999999999991 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.016833929517753088, "auroc_ci": [ -0.05983320846856368, 0.022934508295743432 ], "acc_diff": -9.999999999999998, "acc_ci": [ -13.805, -6.399999999999995 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.05645535241123467, "auroc_ci": [ 0.03161485509370214, 0.08249296043373379 ], "acc_diff": -3.200000000000003, "acc_ci": [ -5.800000000000005, -0.6000000000000005 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.3022074928230123, "auroc_ci": [ 0.2709474530248363, 0.33088028199854513 ], "acc_diff": 7.806191117092864, "acc_ci": [ 5.854643337819648, 9.623149394347246 ], "n": 1486 }, "tracesafe": { "auroc_diff": 0.022628600823045186, "auroc_ci": [ -0.03683144915947646, 0.08031667119637309 ], "acc_diff": -5.8119658119658135, "acc_ci": [ -8.632478632478623, -3.1623931623931623 ], "n": 1170 } }, "ens_3b_self": { "rjudge": { "auroc_diff": 0.018161683277962304, "auroc_ci": [ 0.004031585198545027, 0.0340638947780219 ], "acc_diff": -0.5253940455341444, "acc_ci": [ -2.981611208406313, 2.1015761821365997 ], "n": 571 }, "assebench_strict": { "auroc_diff": 0.034288676936299, "auroc_ci": [ 0.01404434534526663, 0.05431675893144182 ], "acc_diff": -12.533875338753397, "acc_ci": [ -14.769647696476962, -10.29810298102981 ], "n": 1476 }, "atbench": { "auroc_diff": -0.02400086403110524, "auroc_ci": [ -0.04726670989862959, -0.0013094642967923812 ], "acc_diff": -8.600000000000009, "acc_ci": [ -11.6, -5.700000000000005 ], "n": 1000 }, "atbench_codex": { "auroc_diff": 0.012032000000000154, "auroc_ci": [ -0.01599360437117233, 0.03933030561738009 ], "acc_diff": -4.999999999999993, "acc_ci": [ -8.199999999999996, -1.6000000000000014 ], "n": 500 }, "atbench_claw": { "auroc_diff": -0.03975390832008463, "auroc_ci": [ -0.08410974086632922, 0.0020898519731997233 ], "acc_diff": -17.600000000000005, "acc_ci": [ -21.999999999999996, -13.395000000000028 ], "n": 500 }, "atbench_claw_native": { "auroc_diff": 0.04831577901430839, "auroc_ci": [ 0.02534664119978466, 0.0732890359730846 ], "acc_diff": -13.200000000000001, "acc_ci": [ -16.800000000000004, -9.599999999999998 ], "n": 500 }, "agentdyn": { "auroc_diff": 0.3020445428534456, "auroc_ci": [ 0.27128803419075315, 0.3308670754957477 ], "acc_diff": 6.998654104979806, "acc_ci": [ 5.248990578734858, 8.681022880215345 ], "n": 1486 }, "tracesafe": { "auroc_diff": 0.0034413580246913655, "auroc_ci": [ -0.05552856765928123, 0.06287506172146308 ], "acc_diff": -8.119658119658125, "acc_ci": [ -10.85470085470085, -5.470085470085473 ], "n": 1170 } } }, "claims": { "U0_reproduced": { "rjudge": true, "atbench": true }, "U1_lower_bounds": { "atbench_codex": { "h32_s0": 0.007647562328205493, "h32_s1": -0.014216963202614254, "h32_s2": -0.0017409265232811554 }, "atbench": { "h32_s0": -0.03570004917010014, "h32_s1": -0.040664058974714444, "h32_s2": -0.045602033180343796 } }, "U1_per_set": { "atbench_codex": false, "atbench": false }, "U1_supported": false, "U2_rjudge_lower_bounds_teacher_only": { "teach_s0": 0.011744003450816496, "teach_s1": 0.00915685507969467, "teach_s2": 0.008199521078188624 }, "U2_all_positive": true, "U2_seeds": 3 } }