| { |
| "schema": "pp4.acceptance-summary.v1", |
| "split": "heldout_legal", |
| "views": { |
| "all": { |
| "rounds": 1710, |
| "source_clusters": 24, |
| "requests": 24, |
| "arms": { |
| "dyadic24-k32-t0.7": { |
| "overlap_lower_by_position": [ |
| 0.8067433890672516, |
| 0.6977509081002203, |
| 0.5856897932768941, |
| 0.5041357100412681, |
| 0.43608262989799235 |
| ], |
| "overlap_upper_by_position": [ |
| 0.8074799440777224, |
| 0.6987509776828082, |
| 0.5866537351145567, |
| 0.5053890432614611, |
| 0.4371345908366685 |
| ], |
| "survival_by_position": [ |
| 0.7975906186986398, |
| 0.6084112796658909, |
| 0.42148948763725147, |
| 0.28383645104381694, |
| 0.19418719430656808 |
| ], |
| "EA_by_depth": [ |
| 0.7975906186986398, |
| 1.4060018983645306, |
| 1.827491386001782, |
| 2.1113278370455992, |
| 2.3055150313521673 |
| ], |
| "EA_depth5": 2.3055150313521673, |
| "delta_EA_greedy": 0.2137021658550915, |
| "paired_source_cluster_CI95_delta": [ |
| 0.17537044298278803, |
| 0.25443506996441134 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7476042183948785, |
| 0.6552122419842926, |
| 0.555573549177015, |
| 0.48137977775437224, |
| 0.41843609460076453 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7476055229820617, |
| 0.6552270962578102, |
| 0.5555981548086066, |
| 0.48142961175088234, |
| 0.4184711509681901 |
| ] |
| }, |
| "greedy": { |
| "overlap_lower_by_position": [ |
| 0.7476042183948785, |
| 0.6552122419842926, |
| 0.555573549177015, |
| 0.48137977775437224, |
| 0.41843609460076453 |
| ], |
| "overlap_upper_by_position": [ |
| 0.7476175177841777, |
| 0.6553354891713562, |
| 0.5558340859171311, |
| 0.4818676028369031, |
| 0.41880298525876614 |
| ], |
| "survival_by_position": [ |
| 0.7403508771929824, |
| 0.5497076023391813, |
| 0.3760233918128655, |
| 0.25146198830409355, |
| 0.1742690058479532 |
| ], |
| "EA_by_depth": [ |
| 0.7403508771929824, |
| 1.2900584795321637, |
| 1.6660818713450292, |
| 1.9175438596491228, |
| 2.091812865497076 |
| ], |
| "EA_depth5": 2.091812865497076, |
| "delta_EA_greedy": 0.0, |
| "paired_source_cluster_CI95_delta": [ |
| 0.0, |
| 0.0 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7476042183948785, |
| 0.6552122419842926, |
| 0.555573549177015, |
| 0.48137977775437224, |
| 0.41843609460076453 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7476055229820617, |
| 0.6552270962578102, |
| 0.5555981548086066, |
| 0.48142961175088234, |
| 0.4184711509681901 |
| ] |
| } |
| } |
| }, |
| "family:chronology": { |
| "rounds": 802, |
| "source_clusters": 12, |
| "requests": 12, |
| "arms": { |
| "dyadic24-k32-t0.7": { |
| "overlap_lower_by_position": [ |
| 0.8039206769514815, |
| 0.7233677225272249, |
| 0.6050718540353685, |
| 0.5178611858515729, |
| 0.45500182043253357 |
| ], |
| "overlap_upper_by_position": [ |
| 0.8047516057667429, |
| 0.7247860622984176, |
| 0.6063061665307764, |
| 0.5192477750997432, |
| 0.45648573167307743 |
| ], |
| "survival_by_position": [ |
| 0.8000439974821654, |
| 0.6264947561688963, |
| 0.43742739412319254, |
| 0.2990876248918035, |
| 0.20647351830939728 |
| ], |
| "EA_by_depth": [ |
| 0.8000439974821654, |
| 1.4265387536510616, |
| 1.8639661477742542, |
| 2.163053772666058, |
| 2.369527290975455 |
| ], |
| "EA_depth5": 2.369527290975455, |
| "delta_EA_greedy": 0.23611083212258732, |
| "paired_source_cluster_CI95_delta": [ |
| 0.16316321679521495, |
| 0.3015778018965609 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7364463539308701, |
| 0.6797382821052729, |
| 0.5765793568561861, |
| 0.49041874958554166, |
| 0.43930663311858853 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7364471420335049, |
| 0.6797586823031614, |
| 0.5766046211011969, |
| 0.49045654914066905, |
| 0.4393474693034347 |
| ] |
| }, |
| "greedy": { |
| "overlap_lower_by_position": [ |
| 0.7364463539308701, |
| 0.6797382821052729, |
| 0.5765793568561861, |
| 0.49041874958554166, |
| 0.43930663311858853 |
| ], |
| "overlap_upper_by_position": [ |
| 0.7364516287610611, |
| 0.679901049978041, |
| 0.5768637495202164, |
| 0.4907768475362002, |
| 0.43974199047705065 |
| ], |
| "survival_by_position": [ |
| 0.7369077306733167, |
| 0.5685785536159601, |
| 0.3865336658354115, |
| 0.26184538653366585, |
| 0.17955112219451372 |
| ], |
| "EA_by_depth": [ |
| 0.7369077306733167, |
| 1.3054862842892767, |
| 1.6920199501246882, |
| 1.953865336658354, |
| 2.1334164588528677 |
| ], |
| "EA_depth5": 2.1334164588528677, |
| "delta_EA_greedy": 0.0, |
| "paired_source_cluster_CI95_delta": [ |
| 0.0, |
| 0.0 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7364463539308701, |
| 0.6797382821052729, |
| 0.5765793568561861, |
| 0.49041874958554166, |
| 0.43930663311858853 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7364471420335049, |
| 0.6797586823031614, |
| 0.5766046211011969, |
| 0.49045654914066905, |
| 0.4393474693034347 |
| ] |
| } |
| } |
| }, |
| "family:motion_outline": { |
| "rounds": 908, |
| "source_clusters": 12, |
| "requests": 12, |
| "arms": { |
| "dyadic24-k32-t0.7": { |
| "overlap_lower_by_position": [ |
| 0.8092365775219297, |
| 0.6751246028464121, |
| 0.5685703959990348, |
| 0.49201254748635104, |
| 0.41937206733334226 |
| ], |
| "overlap_upper_by_position": [ |
| 0.8098897759338961, |
| 0.6757552311390653, |
| 0.5692955302733584, |
| 0.49314818099901353, |
| 0.4200425038864481 |
| ], |
| "survival_by_position": [ |
| 0.7954236475704596, |
| 0.5924388698031046, |
| 0.40741217375870004, |
| 0.2703657005745599, |
| 0.1833351768503246 |
| ], |
| "EA_by_depth": [ |
| 0.7954236475704596, |
| 1.3878625173735641, |
| 1.7952746911322641, |
| 2.065640391706824, |
| 2.2489755685571486 |
| ], |
| "EA_depth5": 2.2489755685571486, |
| "delta_EA_greedy": 0.19390948926199458, |
| "paired_source_cluster_CI95_delta": [ |
| 0.1621883615015689, |
| 0.23309372869574377 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7574595127782869, |
| 0.6335493739479203, |
| 0.5370199613370426, |
| 0.4733960162911588, |
| 0.400001984588325 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7574612735555664, |
| 0.6335593297287666, |
| 0.5370439852418035, |
| 0.4734564798273045, |
| 0.40003193587472524 |
| ] |
| }, |
| "greedy": { |
| "overlap_lower_by_position": [ |
| 0.7574595127782869, |
| 0.6335493739479203, |
| 0.5370199613370426, |
| 0.4733960162911588, |
| 0.400001984588325 |
| ], |
| "overlap_upper_by_position": [ |
| 0.7574798999389565, |
| 0.6336377140976104, |
| 0.5372594270959038, |
| 0.4739984241487572, |
| 0.4003084013545104 |
| ], |
| "survival_by_position": [ |
| 0.7433920704845814, |
| 0.5330396475770925, |
| 0.3667400881057269, |
| 0.2422907488986784, |
| 0.1696035242290749 |
| ], |
| "EA_by_depth": [ |
| 0.7433920704845814, |
| 1.276431718061674, |
| 1.643171806167401, |
| 1.8854625550660793, |
| 2.055066079295154 |
| ], |
| "EA_depth5": 2.055066079295154, |
| "delta_EA_greedy": 0.0, |
| "paired_source_cluster_CI95_delta": [ |
| 0.0, |
| 0.0 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7574595127782869, |
| 0.6335493739479203, |
| 0.5370199613370426, |
| 0.4733960162911588, |
| 0.400001984588325 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7574612735555664, |
| 0.6335593297287666, |
| 0.5370439852418035, |
| 0.4734564798273045, |
| 0.40003193587472524 |
| ] |
| } |
| } |
| }, |
| "context:16-32K": { |
| "rounds": 545, |
| "source_clusters": 12, |
| "requests": 12, |
| "arms": { |
| "dyadic24-k32-t0.7": { |
| "overlap_lower_by_position": [ |
| 0.8225753282560437, |
| 0.6969314204474354, |
| 0.5769884681850163, |
| 0.516803419228742, |
| 0.44641806014187274 |
| ], |
| "overlap_upper_by_position": [ |
| 0.822962929937667, |
| 0.6974741530648753, |
| 0.5773833046243314, |
| 0.5174107369893364, |
| 0.44686142879638063 |
| ], |
| "survival_by_position": [ |
| 0.8181752706416695, |
| 0.6254908427977329, |
| 0.43732268833967564, |
| 0.29218407661847207, |
| 0.2032034399542774 |
| ], |
| "EA_by_depth": [ |
| 0.8181752706416695, |
| 1.4436661134394024, |
| 1.8809888017790781, |
| 2.1731728783975504, |
| 2.3763763183518276 |
| ], |
| "EA_depth5": 2.3763763183518276, |
| "delta_EA_greedy": 0.15252310734265295, |
| "paired_source_cluster_CI95_delta": [ |
| 0.10383010893025522, |
| 0.2005272063053255 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7785209318685372, |
| 0.6581132001414002, |
| 0.5454627343338907, |
| 0.5011117310352967, |
| 0.42981018142626715 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7785212674680386, |
| 0.6581245748519888, |
| 0.5454724296197044, |
| 0.5011408876376253, |
| 0.4298222314425304 |
| ] |
| }, |
| "greedy": { |
| "overlap_lower_by_position": [ |
| 0.7785209318685372, |
| 0.6581132001414002, |
| 0.5454627343338907, |
| 0.5011117310352967, |
| 0.42981018142626715 |
| ], |
| "overlap_upper_by_position": [ |
| 0.7785242541001139, |
| 0.6581948671128425, |
| 0.545543130998221, |
| 0.5013480972852155, |
| 0.4299285891947366 |
| ], |
| "survival_by_position": [ |
| 0.7779816513761468, |
| 0.5834862385321101, |
| 0.4073394495412844, |
| 0.26605504587155965, |
| 0.1889908256880734 |
| ], |
| "EA_by_depth": [ |
| 0.7779816513761468, |
| 1.361467889908257, |
| 1.7688073394495416, |
| 2.034862385321101, |
| 2.2238532110091747 |
| ], |
| "EA_depth5": 2.2238532110091747, |
| "delta_EA_greedy": 0.0, |
| "paired_source_cluster_CI95_delta": [ |
| 0.0, |
| 0.0 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7785209318685372, |
| 0.6581132001414002, |
| 0.5454627343338907, |
| 0.5011117310352967, |
| 0.42981018142626715 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7785212674680386, |
| 0.6581245748519888, |
| 0.5454724296197044, |
| 0.5011408876376253, |
| 0.4298222314425304 |
| ] |
| } |
| } |
| }, |
| "context:8-16K": { |
| "rounds": 679, |
| "source_clusters": 19, |
| "requests": 19, |
| "arms": { |
| "dyadic24-k32-t0.7": { |
| "overlap_lower_by_position": [ |
| 0.8109112447553453, |
| 0.707797318478551, |
| 0.6062052200348393, |
| 0.51444367624808, |
| 0.43978024208857264 |
| ], |
| "overlap_upper_by_position": [ |
| 0.8117345403133446, |
| 0.7085537132314071, |
| 0.6072349542122047, |
| 0.5159381946111609, |
| 0.44091951320847844 |
| ], |
| "survival_by_position": [ |
| 0.7936128496610968, |
| 0.5994697903477149, |
| 0.42349629166618186, |
| 0.29544308315054446, |
| 0.20766402299126685 |
| ], |
| "EA_by_depth": [ |
| 0.7936128496610968, |
| 1.3930826400088117, |
| 1.8165789316749934, |
| 2.1120220148255378, |
| 2.3196860378168047 |
| ], |
| "EA_depth5": 2.3196860378168047, |
| "delta_EA_greedy": 0.2416300731629022, |
| "paired_source_cluster_CI95_delta": [ |
| 0.18166123017728725, |
| 0.31049722131817714 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7486600707001608, |
| 0.6642280717120037, |
| 0.5813015774611555, |
| 0.4888370908013049, |
| 0.41589896746479993 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7486621920355063, |
| 0.6642391374734279, |
| 0.581329384559086, |
| 0.4889019306076969, |
| 0.41593656039586346 |
| ] |
| }, |
| "greedy": { |
| "overlap_lower_by_position": [ |
| 0.7486600707001608, |
| 0.6642280717120037, |
| 0.5813015774611555, |
| 0.4888370908013049, |
| 0.41589896746479993 |
| ], |
| "overlap_upper_by_position": [ |
| 0.7486847035572444, |
| 0.6643387221881247, |
| 0.58160294234132, |
| 0.48947575113741654, |
| 0.41632771979260597 |
| ], |
| "survival_by_position": [ |
| 0.7334315169366715, |
| 0.5301914580265096, |
| 0.36818851251840945, |
| 0.2606774668630339, |
| 0.18556701030927836 |
| ], |
| "EA_by_depth": [ |
| 0.7334315169366715, |
| 1.263622974963181, |
| 1.6318114874815906, |
| 1.8924889543446244, |
| 2.0780559646539025 |
| ], |
| "EA_depth5": 2.0780559646539025, |
| "delta_EA_greedy": 0.0, |
| "paired_source_cluster_CI95_delta": [ |
| 0.0, |
| 0.0 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7486600707001608, |
| 0.6642280717120037, |
| 0.5813015774611555, |
| 0.4888370908013049, |
| 0.41589896746479993 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7486621920355063, |
| 0.6642391374734279, |
| 0.581329384559086, |
| 0.4889019306076969, |
| 0.41593656039586346 |
| ] |
| } |
| } |
| }, |
| "context:<8K": { |
| "rounds": 420, |
| "source_clusters": 14, |
| "requests": 14, |
| "arms": { |
| "dyadic24-k32-t0.7": { |
| "overlap_lower_by_position": [ |
| 0.7776392771979407, |
| 0.6795272895832064, |
| 0.5602089254511878, |
| 0.4733740281493341, |
| 0.4211359833225281 |
| ], |
| "overlap_upper_by_position": [ |
| 0.7787564869633996, |
| 0.6816120606317134, |
| 0.5618802799326865, |
| 0.4751937010088456, |
| 0.4228638670980742 |
| ], |
| "survival_by_position": [ |
| 0.7759277625944764, |
| 0.5929112580204344, |
| 0.3949500748127734, |
| 0.2530320489379281, |
| 0.16566913586788518 |
| ], |
| "EA_by_depth": [ |
| 0.7759277625944764, |
| 1.3688390206149108, |
| 1.7637890954276843, |
| 2.016821144365612, |
| 2.1824902802334973 |
| ], |
| "EA_depth5": 2.1824902802334973, |
| "delta_EA_greedy": 0.2586807564239735, |
| "paired_source_cluster_CI95_delta": [ |
| 0.16197123657259327, |
| 0.36856276040941555 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7020703083915643, |
| 0.6294990547962546, |
| 0.5201565711312489, |
| 0.4449596093413527, |
| 0.41064728563196784 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7020704364340432, |
| 0.6295265726144844, |
| 0.520198750552984, |
| 0.4450146295475184, |
| 0.4107120708940964 |
| ] |
| }, |
| "greedy": { |
| "overlap_lower_by_position": [ |
| 0.7020703083915643, |
| 0.6294990547962546, |
| 0.5201565711312489, |
| 0.4449596093413527, |
| 0.41064728563196784 |
| ], |
| "overlap_upper_by_position": [ |
| 0.7020707197240946, |
| 0.6297124415816527, |
| 0.5206195472018106, |
| 0.44557658751943985, |
| 0.41128624054624924 |
| ], |
| "survival_by_position": [ |
| 0.7, |
| 0.5261904761904762, |
| 0.34285714285714286, |
| 0.21428571428571427, |
| 0.14047619047619048 |
| ], |
| "EA_by_depth": [ |
| 0.7, |
| 1.2261904761904763, |
| 1.5690476190476192, |
| 1.7833333333333334, |
| 1.9238095238095239 |
| ], |
| "EA_depth5": 1.9238095238095239, |
| "delta_EA_greedy": 0.0, |
| "paired_source_cluster_CI95_delta": [ |
| 0.0, |
| 0.0 |
| ], |
| "greedy_p_lower_by_position": [ |
| 0.7020703083915643, |
| 0.6294990547962546, |
| 0.5201565711312489, |
| 0.4449596093413527, |
| 0.41064728563196784 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7020704364340432, |
| 0.6295265726144844, |
| 0.520198750552984, |
| 0.4450146295475184, |
| 0.4107120708940964 |
| ] |
| } |
| } |
| }, |
| "context:>=32K": { |
| "rounds": 66, |
| "source_clusters": 1, |
| "requests": 1, |
| "arms": { |
| "dyadic24-k32-t0.7": { |
| "overlap_lower_by_position": [ |
| 0.8183395423248736, |
| 0.7171301187218377, |
| 0.6086323977197038, |
| 0.48924019235965577, |
| 0.40781146931278045 |
| ], |
| "overlap_upper_by_position": [ |
| 0.8186428812043667, |
| 0.7175109343606068, |
| 0.6091171900578182, |
| 0.48974278413964656, |
| 0.4086893634382967 |
| ], |
| "survival_by_position": [ |
| 0.8063886403848366, |
| 0.6580009528670993, |
| 0.43898704927082743, |
| 0.2915256896168187, |
| 0.16256543656921563 |
| ], |
| "EA_by_depth": [ |
| 0.8063886403848366, |
| 1.4643895932519357, |
| 1.9033766425227632, |
| 2.194902332139582, |
| 2.3574677687087977 |
| ], |
| "EA_depth5": 2.3574677687087977, |
| "delta_EA_greedy": 0.14534655658758577, |
| "paired_source_cluster_CI95_delta": null, |
| "greedy_p_lower_by_position": [ |
| 0.7712058796518665, |
| 0.70213296983636, |
| 0.5997583016587343, |
| 0.47348645406493406, |
| 0.40018051539342175 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7712142702248057, |
| 0.7021349464239479, |
| 0.5997612610925127, |
| 0.4735196521048376, |
| 0.400190268713852 |
| ] |
| }, |
| "greedy": { |
| "overlap_lower_by_position": [ |
| 0.7712058796518665, |
| 0.70213296983636, |
| 0.5997583016587343, |
| 0.47348645406493406, |
| 0.40018051539342175 |
| ], |
| "overlap_upper_by_position": [ |
| 0.7712669837407986, |
| 0.7021555466134711, |
| 0.5997980734810117, |
| 0.47367706136653354, |
| 0.400231528981128 |
| ], |
| "survival_by_position": [ |
| 0.7575757575757576, |
| 0.6212121212121212, |
| 0.4090909090909091, |
| 0.2727272727272727, |
| 0.15151515151515152 |
| ], |
| "EA_by_depth": [ |
| 0.7575757575757576, |
| 1.378787878787879, |
| 1.787878787878788, |
| 2.0606060606060606, |
| 2.212121212121212 |
| ], |
| "EA_depth5": 2.212121212121212, |
| "delta_EA_greedy": 0.0, |
| "paired_source_cluster_CI95_delta": null, |
| "greedy_p_lower_by_position": [ |
| 0.7712058796518665, |
| 0.70213296983636, |
| 0.5997583016587343, |
| 0.47348645406493406, |
| 0.40018051539342175 |
| ], |
| "greedy_p_upper_by_position": [ |
| 0.7712142702248057, |
| 0.7021349464239479, |
| 0.5997612610925127, |
| 0.4735196521048376, |
| 0.400190268713852 |
| ] |
| } |
| } |
| } |
| }, |
| "family_deltas": { |
| "dyadic24-k32-t0.7": [ |
| 0.23611083212258732, |
| 0.19390948926199458 |
| ], |
| "greedy": [ |
| 0.0, |
| 0.0 |
| ] |
| }, |
| "family_uncertainty": "two held-out task families: report both; source-cluster CIs are conditional on these families", |
| "scope": "one-round expectation on fixed target-history anchors, terminal suffix censored; not long-run throughput", |
| "classification": "MEASURED emitted GPU q on frozen heldout anchors", |
| "frozen_commit": "e1befc9aaa081a4f6cccbb11423a747fcd392503", |
| "frozen_sha256": { |
| "dev-selection.json": "72989e3c7c2bd3ab9d489d8457e02e89a1bd7fbdc1c298f52fa21715f70615be", |
| "heldout-replay.json": "36435b607ca8c56de4ffc539c49926a50085ef4ec33071f645387c2561be0a99", |
| "numeric.py": "c47cb3dfae376b8ffed48ca6c283a0a7dd9d1cfbc5d1b6e1a91b45aa8b28dcb2", |
| "heldout-summary.json": "e95d0b35095db74b6886a2c52e84589391fa87144eb3e8f61414dbbe88311197", |
| "summarize.py": "070ff608f389b107386f1020731a169d7c5785e2b01adac922b4109dae5f62e7" |
| }, |
| "comparisons": { |
| "all": { |
| "sampled_retained_tokens_per_round": 3.3055150313521673, |
| "greedy_retained_tokens_per_round": 3.091812865497076, |
| "delta_retained_tokens_per_round": 0.2137021658550915, |
| "CI95_delta": [ |
| 0.17537044298278803, |
| 0.25443506996441134 |
| ], |
| "PP2_delta": 0.21370155282867476, |
| "PP2_CI95_delta": [ |
| 0.17536962538770962, |
| 0.2544344155783623 |
| ], |
| "change_from_PP2": 6.130264167403254e-07, |
| "PP2_greedy_retained_tokens_per_round": 3.091812865497076, |
| "greedy_change_from_PP2": 0.0, |
| "CI95_greedy_change_from_PP2": [ |
| 0.0, |
| 0.0 |
| ], |
| "sampled_delta_frozen_greedy": 0.2137021658550915, |
| "CI95_sampled_delta_frozen_greedy": [ |
| 0.17537044298278803, |
| 0.25443506996441134 |
| ], |
| "point_within_PP2_interval": true |
| }, |
| "family:chronology": { |
| "sampled_retained_tokens_per_round": 3.369527290975455, |
| "greedy_retained_tokens_per_round": 3.1334164588528677, |
| "delta_retained_tokens_per_round": 0.23611083212258732, |
| "CI95_delta": [ |
| 0.16316321679521495, |
| 0.3015778018965609 |
| ], |
| "PP2_delta": 0.23611027063404277, |
| "PP2_CI95_delta": [ |
| 0.16316342853479984, |
| 0.30157807241611817 |
| ], |
| "change_from_PP2": 5.614885445481832e-07, |
| "PP2_greedy_retained_tokens_per_round": 3.1334164588528677, |
| "greedy_change_from_PP2": 0.0, |
| "CI95_greedy_change_from_PP2": [ |
| 0.0, |
| 0.0 |
| ], |
| "sampled_delta_frozen_greedy": 0.23611083212258732, |
| "CI95_sampled_delta_frozen_greedy": [ |
| 0.16316321679521495, |
| 0.3015778018965609 |
| ], |
| "point_within_PP2_interval": true |
| }, |
| "family:motion_outline": { |
| "sampled_retained_tokens_per_round": 3.2489755685571486, |
| "greedy_retained_tokens_per_round": 3.055066079295154, |
| "delta_retained_tokens_per_round": 0.19390948926199458, |
| "CI95_delta": [ |
| 0.1621883615015689, |
| 0.23309372869574377 |
| ], |
| "PP2_delta": 0.19390883071424136, |
| "PP2_CI95_delta": [ |
| 0.16218746176069568, |
| 0.23309337320225615 |
| ], |
| "change_from_PP2": 6.585477532183859e-07, |
| "PP2_greedy_retained_tokens_per_round": 3.055066079295154, |
| "greedy_change_from_PP2": 0.0, |
| "CI95_greedy_change_from_PP2": [ |
| 0.0, |
| 0.0 |
| ], |
| "sampled_delta_frozen_greedy": 0.19390948926199458, |
| "CI95_sampled_delta_frozen_greedy": [ |
| 0.1621883615015689, |
| 0.23309372869574377 |
| ], |
| "point_within_PP2_interval": true |
| }, |
| "context:16-32K": { |
| "sampled_retained_tokens_per_round": 3.3763763183518276, |
| "greedy_retained_tokens_per_round": 3.2238532110091747, |
| "delta_retained_tokens_per_round": 0.15252310734265295, |
| "CI95_delta": [ |
| 0.10383010893025522, |
| 0.2005272063053255 |
| ], |
| "PP2_delta": 0.15252232221234863, |
| "PP2_CI95_delta": [ |
| 0.10382912743606565, |
| 0.20052669076899177 |
| ], |
| "change_from_PP2": 7.851303043260316e-07, |
| "PP2_greedy_retained_tokens_per_round": 3.2238532110091747, |
| "greedy_change_from_PP2": 0.0, |
| "CI95_greedy_change_from_PP2": [ |
| 0.0, |
| 0.0 |
| ], |
| "sampled_delta_frozen_greedy": 0.15252310734265295, |
| "CI95_sampled_delta_frozen_greedy": [ |
| 0.10383010893025522, |
| 0.2005272063053255 |
| ], |
| "point_within_PP2_interval": true |
| }, |
| "context:8-16K": { |
| "sampled_retained_tokens_per_round": 3.3196860378168047, |
| "greedy_retained_tokens_per_round": 3.0780559646539025, |
| "delta_retained_tokens_per_round": 0.2416300731629022, |
| "CI95_delta": [ |
| 0.18166123017728725, |
| 0.31049722131817714 |
| ], |
| "PP2_delta": 0.24162934535999314, |
| "PP2_CI95_delta": [ |
| 0.1816603668086756, |
| 0.3104964738295701 |
| ], |
| "change_from_PP2": 7.278029090507232e-07, |
| "PP2_greedy_retained_tokens_per_round": 3.0780559646539025, |
| "greedy_change_from_PP2": 0.0, |
| "CI95_greedy_change_from_PP2": [ |
| 0.0, |
| 0.0 |
| ], |
| "sampled_delta_frozen_greedy": 0.2416300731629022, |
| "CI95_sampled_delta_frozen_greedy": [ |
| 0.18166123017728725, |
| 0.31049722131817714 |
| ], |
| "point_within_PP2_interval": true |
| }, |
| "context:<8K": { |
| "sampled_retained_tokens_per_round": 3.1824902802334973, |
| "greedy_retained_tokens_per_round": 2.923809523809524, |
| "delta_retained_tokens_per_round": 0.2586807564239735, |
| "CI95_delta": [ |
| 0.16197123657259327, |
| 0.36856276040941555 |
| ], |
| "PP2_delta": 0.2586805596021795, |
| "PP2_CI95_delta": [ |
| 0.16197031912005402, |
| 0.36856286780395014 |
| ], |
| "change_from_PP2": 1.9682179397406685e-07, |
| "PP2_greedy_retained_tokens_per_round": 2.923809523809524, |
| "greedy_change_from_PP2": 0.0, |
| "CI95_greedy_change_from_PP2": [ |
| 0.0, |
| 0.0 |
| ], |
| "sampled_delta_frozen_greedy": 0.2586807564239735, |
| "CI95_sampled_delta_frozen_greedy": [ |
| 0.16197123657259327, |
| 0.36856276040941555 |
| ], |
| "point_within_PP2_interval": true |
| }, |
| "context:>=32K": { |
| "sampled_retained_tokens_per_round": 3.3574677687087977, |
| "greedy_retained_tokens_per_round": 3.212121212121212, |
| "delta_retained_tokens_per_round": 0.14534655658758577, |
| "CI95_delta": null, |
| "PP2_delta": 0.14534589695382039, |
| "PP2_CI95_delta": null, |
| "change_from_PP2": 6.596337653874684e-07, |
| "PP2_greedy_retained_tokens_per_round": 3.212121212121212, |
| "greedy_change_from_PP2": 0.0, |
| "CI95_greedy_change_from_PP2": null, |
| "sampled_delta_frozen_greedy": 0.14534655658758577, |
| "CI95_sampled_delta_frozen_greedy": null, |
| "point_within_PP2_interval": null |
| } |
| }, |
| "greedy_baseline_drift": { |
| "changed_groups": 0, |
| "group_prefix_total_differences": {}, |
| "status": "AGGREGATES_IDENTICAL", |
| "scope": "source/context aggregate totals, not per-anchor identity; same inputs; CPython3.13/Torch2.12 versus PP2 CPython3.12/Torch2.10" |
| }, |
| "input_sha256": "cd18893e5d370d6483247621acf22fe9a56f12c872a531e5d4fa447b37d1ab57" |
| } |
|
|
