| { |
| "GLM-5.3-Flash-JANGHT2.4": { |
| "bundle": "GLM-5.3-Flash-JANGT-A", |
| "assistant": { |
| "positions": 60243, |
| "median_kl": 0.05787832289934158, |
| "mean_kl": 0.19320770706862295, |
| "p90": 0.5197924971580505, |
| "top1_pct": 83.75412911043607 |
| }, |
| "context": { |
| "positions": 7240, |
| "median_kl": 1.2097653150558472, |
| "mean_kl": 2.9719159244851086, |
| "p90": 8.939142227172852, |
| "top1_pct": 46.20165745856354 |
| }, |
| "close": { |
| "n_all": 34, |
| "n_ref_ge_half": 20, |
| "argmax_correct": 18, |
| "p_q_ge_half": 18, |
| "median_p_q": 0.939609944820404, |
| "min_p_q": 0.1804104745388031, |
| "each": [ |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 407, |
| "p_ref": 0.9838244318962097, |
| "p_q": 0.9910188317298889, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 588, |
| "p_ref": 0.7189128398895264, |
| "p_q": 0.700715184211731, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 965, |
| "p_ref": 0.8597419261932373, |
| "p_q": 0.6827294230461121, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 1116, |
| "p_ref": 0.25516748428344727, |
| "p_q": 0.5900624990463257, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 1280, |
| "p_ref": 0.9985171556472778, |
| "p_q": 0.9973294734954834, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 411, |
| "p_ref": 0.09345996379852295, |
| "p_q": 0.015882549807429314, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 481, |
| "p_ref": 0.9994451403617859, |
| "p_q": 0.9953188896179199, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 818, |
| "p_ref": 0.05112185701727867, |
| "p_q": 0.2645888030529022, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 999, |
| "p_ref": 0.010241878218948841, |
| "p_q": 0.0032730763778090477, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 1232, |
| "p_ref": 0.9731510877609253, |
| "p_q": 0.942426323890686, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 408, |
| "p_ref": 0.6796087026596069, |
| "p_q": 0.8086053729057312, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 514, |
| "p_ref": 0.3617189824581146, |
| "p_q": 0.32245180010795593, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 727, |
| "p_ref": 0.5420432090759277, |
| "p_q": 0.3267715275287628, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 899, |
| "p_ref": 0.9765183329582214, |
| "p_q": 0.9367935657501221, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 410, |
| "p_ref": 0.5529681444168091, |
| "p_q": 0.1804104745388031, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 488, |
| "p_ref": 0.743448793888092, |
| "p_q": 0.8236624002456665, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 712, |
| "p_ref": 0.017172181978821754, |
| "p_q": 0.012756268493831158, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1193, |
| "p_ref": 0.013264749199151993, |
| "p_q": 0.025790251791477203, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1447, |
| "p_ref": 0.9964795112609863, |
| "p_q": 0.9963350892066956, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 405, |
| "p_ref": 0.06360142678022385, |
| "p_q": 0.12390553206205368, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 491, |
| "p_ref": 0.09378442913293839, |
| "p_q": 0.006607736926525831, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 717, |
| "p_ref": 0.022891120985150337, |
| "p_q": 0.008860631845891476, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 973, |
| "p_ref": 0.06886352598667145, |
| "p_q": 0.012542438693344593, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 1502, |
| "p_ref": 0.8409368395805359, |
| "p_q": 0.961441159248352, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 408, |
| "p_ref": 0.9598525762557983, |
| "p_q": 0.5953214168548584, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 478, |
| "p_ref": 0.9998569488525391, |
| "p_q": 0.9995366334915161, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 815, |
| "p_ref": 0.025962302461266518, |
| "p_q": 0.019399892538785934, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 982, |
| "p_ref": 0.27114608883857727, |
| "p_q": 0.05650907754898071, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 1361, |
| "p_ref": 0.8944574594497681, |
| "p_q": 0.8842508792877197, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_28_design_low", |
| "pos": 8025, |
| "p_ref": 0.8218483328819275, |
| "p_q": 0.7348714470863342, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_32_coding_low", |
| "pos": 81, |
| "p_ref": 0.9979269504547119, |
| "p_q": 0.9977766275405884, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_34_agentic_high", |
| "pos": 370, |
| "p_ref": 0.9970080256462097, |
| "p_q": 0.9986581206321716, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_35_agentic_low", |
| "pos": 356, |
| "p_ref": 0.9999904632568359, |
| "p_q": 0.9999504089355469, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_36_general_high", |
| "pos": 177, |
| "p_ref": 0.3396732807159424, |
| "p_q": 0.7962968945503235, |
| "argmax_ok": true |
| } |
| ] |
| }, |
| "tool_call": { |
| "n_all": 48, |
| "n_ref_ge_half": 41, |
| "argmax_correct": 40, |
| "p_q_ge_half": 40, |
| "median_p_q": 0.9998798370361328, |
| "min_p_q": 0.07585730403661728, |
| "each": [ |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 408, |
| "p_ref": 0.998909592628479, |
| "p_q": 0.9996071457862854, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 421, |
| "p_ref": 0.4377860128879547, |
| "p_q": 0.18241006135940552, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 589, |
| "p_ref": 0.9998664855957031, |
| "p_q": 0.9995423555374146, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 616, |
| "p_ref": 0.5926499366760254, |
| "p_q": 0.07585730403661728, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 629, |
| "p_ref": 0.05340293049812317, |
| "p_q": 0.04742521792650223, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 966, |
| "p_ref": 0.9999923706054688, |
| "p_q": 0.9998703002929688, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 1117, |
| "p_ref": 0.9998550415039062, |
| "p_q": 0.9999275207519531, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 1281, |
| "p_ref": 0.979510486125946, |
| "p_q": 0.8921334147453308, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 412, |
| "p_ref": 0.0736844465136528, |
| "p_q": 0.6063386797904968, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 482, |
| "p_ref": 0.9999847412109375, |
| "p_q": 0.9999828338623047, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 495, |
| "p_ref": 0.9960519671440125, |
| "p_q": 0.9979250431060791, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 508, |
| "p_ref": 0.9968274235725403, |
| "p_q": 0.9991934895515442, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 840, |
| "p_ref": 0.9999809265136719, |
| "p_q": 0.9999370574951172, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 1020, |
| "p_ref": 0.9999561309814453, |
| "p_q": 0.9999713897705078, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 1251, |
| "p_ref": 1.0, |
| "p_q": 0.9999980926513672, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 409, |
| "p_ref": 0.9621272683143616, |
| "p_q": 0.9957537651062012, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 515, |
| "p_ref": 0.9767139554023743, |
| "p_q": 0.9850862622261047, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 525, |
| "p_ref": 0.1191921979188919, |
| "p_q": 0.02595403417944908, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 728, |
| "p_ref": 0.9975558519363403, |
| "p_q": 0.9975216388702393, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 419, |
| "p_ref": 0.9999980926513672, |
| "p_q": 0.9999942779541016, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 489, |
| "p_ref": 0.9332356452941895, |
| "p_q": 0.9976966977119446, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 500, |
| "p_ref": 0.9999313354492188, |
| "p_q": 0.9999713897705078, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 744, |
| "p_ref": 0.999969482421875, |
| "p_q": 0.9999618530273438, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 757, |
| "p_ref": 0.9999675750732422, |
| "p_q": 0.9999942779541016, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 770, |
| "p_ref": 0.9924128651618958, |
| "p_q": 0.9990677237510681, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 783, |
| "p_ref": 0.9525591731071472, |
| "p_q": 0.9323834180831909, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1219, |
| "p_ref": 0.9999675750732422, |
| "p_q": 0.9999847412109375, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1448, |
| "p_ref": 0.06318042427301407, |
| "p_q": 0.0798255130648613, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1461, |
| "p_ref": 0.9982962608337402, |
| "p_q": 0.9999542236328125, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1474, |
| "p_ref": 0.9889983534812927, |
| "p_q": 0.9977975487709045, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 422, |
| "p_ref": 0.999969482421875, |
| "p_q": 0.9999504089355469, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 492, |
| "p_ref": 0.8776340484619141, |
| "p_q": 0.9677923321723938, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 505, |
| "p_ref": 0.946465015411377, |
| "p_q": 0.9625898003578186, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 748, |
| "p_ref": 0.9997082352638245, |
| "p_q": 0.9998912811279297, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 761, |
| "p_ref": 0.9995595216751099, |
| "p_q": 0.99993896484375, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 994, |
| "p_ref": 0.9999771118164062, |
| "p_q": 0.9999866485595703, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 1009, |
| "p_ref": 0.9999809265136719, |
| "p_q": 0.9999961853027344, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 1022, |
| "p_ref": 0.9768145084381104, |
| "p_q": 0.9963939785957336, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 1035, |
| "p_ref": 0.320767343044281, |
| "p_q": 0.02297690138220787, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 1532, |
| "p_ref": 0.9999961853027344, |
| "p_q": 0.9999980926513672, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 409, |
| "p_ref": 0.9866337776184082, |
| "p_q": 0.9788455963134766, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 479, |
| "p_ref": 0.9999980926513672, |
| "p_q": 0.9999828338623047, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 490, |
| "p_ref": 0.9999637603759766, |
| "p_q": 0.9999370574951172, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 503, |
| "p_ref": 0.9525591731071472, |
| "p_q": 0.9980716109275818, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 832, |
| "p_ref": 0.9999046325683594, |
| "p_q": 0.9998798370361328, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 1030, |
| "p_ref": 0.0414167195558548, |
| "p_q": 0.15861880779266357, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_34_agentic_high", |
| "pos": 381, |
| "p_ref": 0.9999942779541016, |
| "p_q": 0.9999885559082031, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_35_agentic_low", |
| "pos": 357, |
| "p_ref": 0.9996204972267151, |
| "p_q": 0.9997444748878479, |
| "argmax_ok": true |
| } |
| ] |
| } |
| }, |
| "GLM-5.3-Flash-JANGH2_revision2": { |
| "bundle": "GLM-5.3-Flash-JANGH2-r2", |
| "assistant": { |
| "positions": 60243, |
| "median_kl": 0.06881622225046158, |
| "mean_kl": 0.21784130269915855, |
| "p90": 0.5812361598014831, |
| "top1_pct": 82.5938283286025 |
| }, |
| "context": { |
| "positions": 7240, |
| "median_kl": 1.4476576447486877, |
| "mean_kl": 3.102558822176583, |
| "p90": 9.103206443786622, |
| "top1_pct": 45.027624309392266 |
| }, |
| "close": { |
| "n_all": 34, |
| "n_ref_ge_half": 20, |
| "argmax_correct": 14, |
| "p_q_ge_half": 12, |
| "median_p_q": 0.8495109677314758, |
| "min_p_q": 0.05205969512462616, |
| "each": [ |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 407, |
| "p_ref": 0.9838244318962097, |
| "p_q": 0.2916111350059509, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 588, |
| "p_ref": 0.7189128398895264, |
| "p_q": 0.3790413737297058, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 965, |
| "p_ref": 0.8597419261932373, |
| "p_q": 0.299537718296051, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 1116, |
| "p_ref": 0.25516748428344727, |
| "p_q": 0.20701055228710175, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 1280, |
| "p_ref": 0.9985171556472778, |
| "p_q": 0.9975996017456055, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 411, |
| "p_ref": 0.09345996379852295, |
| "p_q": 0.0008723821956664324, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 481, |
| "p_ref": 0.9994451403617859, |
| "p_q": 0.9887097477912903, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 818, |
| "p_ref": 0.05112185701727867, |
| "p_q": 0.01732826791703701, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 999, |
| "p_ref": 0.010241878218948841, |
| "p_q": 0.016675179824233055, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 1232, |
| "p_ref": 0.9731510877609253, |
| "p_q": 0.9661397933959961, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 408, |
| "p_ref": 0.6796087026596069, |
| "p_q": 0.11788290739059448, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 514, |
| "p_ref": 0.3617189824581146, |
| "p_q": 0.1208728477358818, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 727, |
| "p_ref": 0.5420432090759277, |
| "p_q": 0.14295782148838043, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 899, |
| "p_ref": 0.9765183329582214, |
| "p_q": 0.996669590473175, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 410, |
| "p_ref": 0.5529681444168091, |
| "p_q": 0.05205969512462616, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 488, |
| "p_ref": 0.743448793888092, |
| "p_q": 0.22527901828289032, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 712, |
| "p_ref": 0.017172181978821754, |
| "p_q": 0.016983453184366226, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1193, |
| "p_ref": 0.013264749199151993, |
| "p_q": 0.02826460637152195, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1447, |
| "p_ref": 0.9964795112609863, |
| "p_q": 0.9916768074035645, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 405, |
| "p_ref": 0.06360142678022385, |
| "p_q": 0.00022873509442433715, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 491, |
| "p_ref": 0.09378442913293839, |
| "p_q": 0.06594368815422058, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 717, |
| "p_ref": 0.022891120985150337, |
| "p_q": 0.007983765564858913, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 973, |
| "p_ref": 0.06886352598667145, |
| "p_q": 0.010131229646503925, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 1502, |
| "p_ref": 0.8409368395805359, |
| "p_q": 0.907599151134491, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 408, |
| "p_ref": 0.9598525762557983, |
| "p_q": 0.46985504031181335, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 478, |
| "p_ref": 0.9998569488525391, |
| "p_q": 0.9906918406486511, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 815, |
| "p_ref": 0.025962302461266518, |
| "p_q": 0.02887396141886711, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 982, |
| "p_ref": 0.27114608883857727, |
| "p_q": 0.19489246606826782, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 1361, |
| "p_ref": 0.8944574594497681, |
| "p_q": 0.7914227843284607, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_28_design_low", |
| "pos": 8025, |
| "p_ref": 0.8218483328819275, |
| "p_q": 0.7522467970848083, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_32_coding_low", |
| "pos": 81, |
| "p_ref": 0.9979269504547119, |
| "p_q": 0.9863665699958801, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_34_agentic_high", |
| "pos": 370, |
| "p_ref": 0.9970080256462097, |
| "p_q": 0.9614063501358032, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_35_agentic_low", |
| "pos": 356, |
| "p_ref": 0.9999904632568359, |
| "p_q": 0.9999923706054688, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_36_general_high", |
| "pos": 177, |
| "p_ref": 0.3396732807159424, |
| "p_q": 0.36337098479270935, |
| "argmax_ok": false |
| } |
| ] |
| }, |
| "tool_call": { |
| "n_all": 48, |
| "n_ref_ge_half": 41, |
| "argmax_correct": 40, |
| "p_q_ge_half": 40, |
| "median_p_q": 0.9998245239257812, |
| "min_p_q": 0.164503276348114, |
| "each": [ |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 408, |
| "p_ref": 0.998909592628479, |
| "p_q": 0.9371992349624634, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 421, |
| "p_ref": 0.4377860128879547, |
| "p_q": 0.24495118856430054, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 589, |
| "p_ref": 0.9998664855957031, |
| "p_q": 0.9995080232620239, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 616, |
| "p_ref": 0.5926499366760254, |
| "p_q": 0.164503276348114, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 629, |
| "p_ref": 0.05340293049812317, |
| "p_q": 0.02033219300210476, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 966, |
| "p_ref": 0.9999923706054688, |
| "p_q": 0.9998531341552734, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 1117, |
| "p_ref": 0.9998550415039062, |
| "p_q": 0.9999332427978516, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n00_agentic_low", |
| "pos": 1281, |
| "p_ref": 0.979510486125946, |
| "p_q": 0.9229016304016113, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 412, |
| "p_ref": 0.0736844465136528, |
| "p_q": 0.057815179228782654, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 482, |
| "p_ref": 0.9999847412109375, |
| "p_q": 0.9998798370361328, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 495, |
| "p_ref": 0.9960519671440125, |
| "p_q": 0.9999198913574219, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 508, |
| "p_ref": 0.9968274235725403, |
| "p_q": 0.9998416900634766, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 840, |
| "p_ref": 0.9999809265136719, |
| "p_q": 0.9997101426124573, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 1020, |
| "p_ref": 0.9999561309814453, |
| "p_q": 0.9999122619628906, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n01_agentic_high", |
| "pos": 1251, |
| "p_ref": 1.0, |
| "p_q": 0.9999980926513672, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 409, |
| "p_ref": 0.9621272683143616, |
| "p_q": 0.8799203038215637, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 515, |
| "p_ref": 0.9767139554023743, |
| "p_q": 0.9765891432762146, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 525, |
| "p_ref": 0.1191921979188919, |
| "p_q": 0.005220085848122835, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n02_agentic_low", |
| "pos": 728, |
| "p_ref": 0.9975558519363403, |
| "p_q": 0.9956303238868713, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 419, |
| "p_ref": 0.9999980926513672, |
| "p_q": 0.9999847412109375, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 489, |
| "p_ref": 0.9332356452941895, |
| "p_q": 0.5371268391609192, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 500, |
| "p_ref": 0.9999313354492188, |
| "p_q": 0.9999961853027344, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 744, |
| "p_ref": 0.999969482421875, |
| "p_q": 0.9998912811279297, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 757, |
| "p_ref": 0.9999675750732422, |
| "p_q": 0.9999961853027344, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 770, |
| "p_ref": 0.9924128651618958, |
| "p_q": 0.999079167842865, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 783, |
| "p_ref": 0.9525591731071472, |
| "p_q": 0.9840852618217468, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1219, |
| "p_ref": 0.9999675750732422, |
| "p_q": 0.9999217987060547, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1448, |
| "p_ref": 0.06318042427301407, |
| "p_q": 0.1402663141489029, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1461, |
| "p_ref": 0.9982962608337402, |
| "p_q": 0.9998245239257812, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n03_agentic_high", |
| "pos": 1474, |
| "p_ref": 0.9889983534812927, |
| "p_q": 0.9989610314369202, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 422, |
| "p_ref": 0.999969482421875, |
| "p_q": 0.9999313354492188, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 492, |
| "p_ref": 0.8776340484619141, |
| "p_q": 0.928158700466156, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 505, |
| "p_ref": 0.946465015411377, |
| "p_q": 0.9958600997924805, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 748, |
| "p_ref": 0.9997082352638245, |
| "p_q": 0.9995251893997192, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 761, |
| "p_ref": 0.9995595216751099, |
| "p_q": 0.9998397827148438, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 994, |
| "p_ref": 0.9999771118164062, |
| "p_q": 0.9999160766601562, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 1009, |
| "p_ref": 0.9999809265136719, |
| "p_q": 0.9999961853027344, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 1022, |
| "p_ref": 0.9768145084381104, |
| "p_q": 0.9889700412750244, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 1035, |
| "p_ref": 0.320767343044281, |
| "p_q": 0.622435450553894, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n04_agentic_max", |
| "pos": 1532, |
| "p_ref": 0.9999961853027344, |
| "p_q": 0.9999923706054688, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 409, |
| "p_ref": 0.9866337776184082, |
| "p_q": 0.9518871903419495, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 479, |
| "p_ref": 0.9999980926513672, |
| "p_q": 0.9999504089355469, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 490, |
| "p_ref": 0.9999637603759766, |
| "p_q": 0.9999866485595703, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 503, |
| "p_ref": 0.9525591731071472, |
| "p_q": 0.7057181596755981, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 832, |
| "p_ref": 0.9999046325683594, |
| "p_q": 0.9996529221534729, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_n05_agentic_low", |
| "pos": 1030, |
| "p_ref": 0.0414167195558548, |
| "p_q": 0.1559947282075882, |
| "argmax_ok": false |
| }, |
| { |
| "seq": "held_34_agentic_high", |
| "pos": 381, |
| "p_ref": 0.9999942779541016, |
| "p_q": 0.9999866485595703, |
| "argmax_ok": true |
| }, |
| { |
| "seq": "held_35_agentic_low", |
| "pos": 357, |
| "p_ref": 0.9996204972267151, |
| "p_q": 0.9953587651252747, |
| "argmax_ok": true |
| } |
| ] |
| } |
| } |
| } |
