{ "schema": "pp4.acceptance-summary.v1", "split": "heldout_legal", "views": { "all": { "rounds": 1710, "source_clusters": 24, "requests": 24, "arms": { "dyadic24-k32-t0.7": { "overlap_lower_by_position": [ 0.8067433890672516, 0.6977509081002203, 0.5856897932768941, 0.5041357100412681, 0.43608262989799235 ], "overlap_upper_by_position": [ 0.8074799440777224, 0.6987509776828082, 0.5866537351145567, 0.5053890432614611, 0.4371345908366685 ], "survival_by_position": [ 0.7975906186986398, 0.6084112796658909, 0.42148948763725147, 0.28383645104381694, 0.19418719430656808 ], "EA_by_depth": [ 0.7975906186986398, 1.4060018983645306, 1.827491386001782, 2.1113278370455992, 2.3055150313521673 ], "EA_depth5": 2.3055150313521673, "delta_EA_greedy": 0.2137021658550915, "paired_source_cluster_CI95_delta": [ 0.17537044298278803, 0.25443506996441134 ], "greedy_p_lower_by_position": [ 0.7476042183948785, 0.6552122419842926, 0.555573549177015, 0.48137977775437224, 0.41843609460076453 ], "greedy_p_upper_by_position": [ 0.7476055229820617, 0.6552270962578102, 0.5555981548086066, 0.48142961175088234, 0.4184711509681901 ] }, "greedy": { "overlap_lower_by_position": [ 0.7476042183948785, 0.6552122419842926, 0.555573549177015, 0.48137977775437224, 0.41843609460076453 ], "overlap_upper_by_position": [ 0.7476175177841777, 0.6553354891713562, 0.5558340859171311, 0.4818676028369031, 0.41880298525876614 ], "survival_by_position": [ 0.7403508771929824, 0.5497076023391813, 0.3760233918128655, 0.25146198830409355, 0.1742690058479532 ], "EA_by_depth": [ 0.7403508771929824, 1.2900584795321637, 1.6660818713450292, 1.9175438596491228, 2.091812865497076 ], "EA_depth5": 2.091812865497076, "delta_EA_greedy": 0.0, "paired_source_cluster_CI95_delta": [ 0.0, 0.0 ], "greedy_p_lower_by_position": [ 0.7476042183948785, 0.6552122419842926, 0.555573549177015, 0.48137977775437224, 0.41843609460076453 ], "greedy_p_upper_by_position": [ 0.7476055229820617, 0.6552270962578102, 0.5555981548086066, 0.48142961175088234, 0.4184711509681901 ] } } }, "family:chronology": { "rounds": 802, "source_clusters": 12, "requests": 12, "arms": { "dyadic24-k32-t0.7": { "overlap_lower_by_position": [ 0.8039206769514815, 0.7233677225272249, 0.6050718540353685, 0.5178611858515729, 0.45500182043253357 ], "overlap_upper_by_position": [ 0.8047516057667429, 0.7247860622984176, 0.6063061665307764, 0.5192477750997432, 0.45648573167307743 ], "survival_by_position": [ 0.8000439974821654, 0.6264947561688963, 0.43742739412319254, 0.2990876248918035, 0.20647351830939728 ], "EA_by_depth": [ 0.8000439974821654, 1.4265387536510616, 1.8639661477742542, 2.163053772666058, 2.369527290975455 ], "EA_depth5": 2.369527290975455, "delta_EA_greedy": 0.23611083212258732, "paired_source_cluster_CI95_delta": [ 0.16316321679521495, 0.3015778018965609 ], "greedy_p_lower_by_position": [ 0.7364463539308701, 0.6797382821052729, 0.5765793568561861, 0.49041874958554166, 0.43930663311858853 ], "greedy_p_upper_by_position": [ 0.7364471420335049, 0.6797586823031614, 0.5766046211011969, 0.49045654914066905, 0.4393474693034347 ] }, "greedy": { "overlap_lower_by_position": [ 0.7364463539308701, 0.6797382821052729, 0.5765793568561861, 0.49041874958554166, 0.43930663311858853 ], "overlap_upper_by_position": [ 0.7364516287610611, 0.679901049978041, 0.5768637495202164, 0.4907768475362002, 0.43974199047705065 ], "survival_by_position": [ 0.7369077306733167, 0.5685785536159601, 0.3865336658354115, 0.26184538653366585, 0.17955112219451372 ], "EA_by_depth": [ 0.7369077306733167, 1.3054862842892767, 1.6920199501246882, 1.953865336658354, 2.1334164588528677 ], "EA_depth5": 2.1334164588528677, "delta_EA_greedy": 0.0, "paired_source_cluster_CI95_delta": [ 0.0, 0.0 ], "greedy_p_lower_by_position": [ 0.7364463539308701, 0.6797382821052729, 0.5765793568561861, 0.49041874958554166, 0.43930663311858853 ], "greedy_p_upper_by_position": [ 0.7364471420335049, 0.6797586823031614, 0.5766046211011969, 0.49045654914066905, 0.4393474693034347 ] } } }, "family:motion_outline": { "rounds": 908, "source_clusters": 12, "requests": 12, "arms": { "dyadic24-k32-t0.7": { "overlap_lower_by_position": [ 0.8092365775219297, 0.6751246028464121, 0.5685703959990348, 0.49201254748635104, 0.41937206733334226 ], "overlap_upper_by_position": [ 0.8098897759338961, 0.6757552311390653, 0.5692955302733584, 0.49314818099901353, 0.4200425038864481 ], "survival_by_position": [ 0.7954236475704596, 0.5924388698031046, 0.40741217375870004, 0.2703657005745599, 0.1833351768503246 ], "EA_by_depth": [ 0.7954236475704596, 1.3878625173735641, 1.7952746911322641, 2.065640391706824, 2.2489755685571486 ], "EA_depth5": 2.2489755685571486, "delta_EA_greedy": 0.19390948926199458, "paired_source_cluster_CI95_delta": [ 0.1621883615015689, 0.23309372869574377 ], "greedy_p_lower_by_position": [ 0.7574595127782869, 0.6335493739479203, 0.5370199613370426, 0.4733960162911588, 0.400001984588325 ], "greedy_p_upper_by_position": [ 0.7574612735555664, 0.6335593297287666, 0.5370439852418035, 0.4734564798273045, 0.40003193587472524 ] }, "greedy": { "overlap_lower_by_position": [ 0.7574595127782869, 0.6335493739479203, 0.5370199613370426, 0.4733960162911588, 0.400001984588325 ], "overlap_upper_by_position": [ 0.7574798999389565, 0.6336377140976104, 0.5372594270959038, 0.4739984241487572, 0.4003084013545104 ], "survival_by_position": [ 0.7433920704845814, 0.5330396475770925, 0.3667400881057269, 0.2422907488986784, 0.1696035242290749 ], "EA_by_depth": [ 0.7433920704845814, 1.276431718061674, 1.643171806167401, 1.8854625550660793, 2.055066079295154 ], "EA_depth5": 2.055066079295154, "delta_EA_greedy": 0.0, "paired_source_cluster_CI95_delta": [ 0.0, 0.0 ], "greedy_p_lower_by_position": [ 0.7574595127782869, 0.6335493739479203, 0.5370199613370426, 0.4733960162911588, 0.400001984588325 ], "greedy_p_upper_by_position": [ 0.7574612735555664, 0.6335593297287666, 0.5370439852418035, 0.4734564798273045, 0.40003193587472524 ] } } }, "context:16-32K": { "rounds": 545, "source_clusters": 12, "requests": 12, "arms": { "dyadic24-k32-t0.7": { "overlap_lower_by_position": [ 0.8225753282560437, 0.6969314204474354, 0.5769884681850163, 0.516803419228742, 0.44641806014187274 ], "overlap_upper_by_position": [ 0.822962929937667, 0.6974741530648753, 0.5773833046243314, 0.5174107369893364, 0.44686142879638063 ], "survival_by_position": [ 0.8181752706416695, 0.6254908427977329, 0.43732268833967564, 0.29218407661847207, 0.2032034399542774 ], "EA_by_depth": [ 0.8181752706416695, 1.4436661134394024, 1.8809888017790781, 2.1731728783975504, 2.3763763183518276 ], "EA_depth5": 2.3763763183518276, "delta_EA_greedy": 0.15252310734265295, "paired_source_cluster_CI95_delta": [ 0.10383010893025522, 0.2005272063053255 ], "greedy_p_lower_by_position": [ 0.7785209318685372, 0.6581132001414002, 0.5454627343338907, 0.5011117310352967, 0.42981018142626715 ], "greedy_p_upper_by_position": [ 0.7785212674680386, 0.6581245748519888, 0.5454724296197044, 0.5011408876376253, 0.4298222314425304 ] }, "greedy": { "overlap_lower_by_position": [ 0.7785209318685372, 0.6581132001414002, 0.5454627343338907, 0.5011117310352967, 0.42981018142626715 ], "overlap_upper_by_position": [ 0.7785242541001139, 0.6581948671128425, 0.545543130998221, 0.5013480972852155, 0.4299285891947366 ], "survival_by_position": [ 0.7779816513761468, 0.5834862385321101, 0.4073394495412844, 0.26605504587155965, 0.1889908256880734 ], "EA_by_depth": [ 0.7779816513761468, 1.361467889908257, 1.7688073394495416, 2.034862385321101, 2.2238532110091747 ], "EA_depth5": 2.2238532110091747, "delta_EA_greedy": 0.0, "paired_source_cluster_CI95_delta": [ 0.0, 0.0 ], "greedy_p_lower_by_position": [ 0.7785209318685372, 0.6581132001414002, 0.5454627343338907, 0.5011117310352967, 0.42981018142626715 ], "greedy_p_upper_by_position": [ 0.7785212674680386, 0.6581245748519888, 0.5454724296197044, 0.5011408876376253, 0.4298222314425304 ] } } }, "context:8-16K": { "rounds": 679, "source_clusters": 19, "requests": 19, "arms": { "dyadic24-k32-t0.7": { "overlap_lower_by_position": [ 0.8109112447553453, 0.707797318478551, 0.6062052200348393, 0.51444367624808, 0.43978024208857264 ], "overlap_upper_by_position": [ 0.8117345403133446, 0.7085537132314071, 0.6072349542122047, 0.5159381946111609, 0.44091951320847844 ], "survival_by_position": [ 0.7936128496610968, 0.5994697903477149, 0.42349629166618186, 0.29544308315054446, 0.20766402299126685 ], "EA_by_depth": [ 0.7936128496610968, 1.3930826400088117, 1.8165789316749934, 2.1120220148255378, 2.3196860378168047 ], "EA_depth5": 2.3196860378168047, "delta_EA_greedy": 0.2416300731629022, "paired_source_cluster_CI95_delta": [ 0.18166123017728725, 0.31049722131817714 ], "greedy_p_lower_by_position": [ 0.7486600707001608, 0.6642280717120037, 0.5813015774611555, 0.4888370908013049, 0.41589896746479993 ], "greedy_p_upper_by_position": [ 0.7486621920355063, 0.6642391374734279, 0.581329384559086, 0.4889019306076969, 0.41593656039586346 ] }, "greedy": { "overlap_lower_by_position": [ 0.7486600707001608, 0.6642280717120037, 0.5813015774611555, 0.4888370908013049, 0.41589896746479993 ], "overlap_upper_by_position": [ 0.7486847035572444, 0.6643387221881247, 0.58160294234132, 0.48947575113741654, 0.41632771979260597 ], "survival_by_position": [ 0.7334315169366715, 0.5301914580265096, 0.36818851251840945, 0.2606774668630339, 0.18556701030927836 ], "EA_by_depth": [ 0.7334315169366715, 1.263622974963181, 1.6318114874815906, 1.8924889543446244, 2.0780559646539025 ], "EA_depth5": 2.0780559646539025, "delta_EA_greedy": 0.0, "paired_source_cluster_CI95_delta": [ 0.0, 0.0 ], "greedy_p_lower_by_position": [ 0.7486600707001608, 0.6642280717120037, 0.5813015774611555, 0.4888370908013049, 0.41589896746479993 ], "greedy_p_upper_by_position": [ 0.7486621920355063, 0.6642391374734279, 0.581329384559086, 0.4889019306076969, 0.41593656039586346 ] } } }, "context:<8K": { "rounds": 420, "source_clusters": 14, "requests": 14, "arms": { "dyadic24-k32-t0.7": { "overlap_lower_by_position": [ 0.7776392771979407, 0.6795272895832064, 0.5602089254511878, 0.4733740281493341, 0.4211359833225281 ], "overlap_upper_by_position": [ 0.7787564869633996, 0.6816120606317134, 0.5618802799326865, 0.4751937010088456, 0.4228638670980742 ], "survival_by_position": [ 0.7759277625944764, 0.5929112580204344, 0.3949500748127734, 0.2530320489379281, 0.16566913586788518 ], "EA_by_depth": [ 0.7759277625944764, 1.3688390206149108, 1.7637890954276843, 2.016821144365612, 2.1824902802334973 ], "EA_depth5": 2.1824902802334973, "delta_EA_greedy": 0.2586807564239735, "paired_source_cluster_CI95_delta": [ 0.16197123657259327, 0.36856276040941555 ], "greedy_p_lower_by_position": [ 0.7020703083915643, 0.6294990547962546, 0.5201565711312489, 0.4449596093413527, 0.41064728563196784 ], "greedy_p_upper_by_position": [ 0.7020704364340432, 0.6295265726144844, 0.520198750552984, 0.4450146295475184, 0.4107120708940964 ] }, "greedy": { "overlap_lower_by_position": [ 0.7020703083915643, 0.6294990547962546, 0.5201565711312489, 0.4449596093413527, 0.41064728563196784 ], "overlap_upper_by_position": [ 0.7020707197240946, 0.6297124415816527, 0.5206195472018106, 0.44557658751943985, 0.41128624054624924 ], "survival_by_position": [ 0.7, 0.5261904761904762, 0.34285714285714286, 0.21428571428571427, 0.14047619047619048 ], "EA_by_depth": [ 0.7, 1.2261904761904763, 1.5690476190476192, 1.7833333333333334, 1.9238095238095239 ], "EA_depth5": 1.9238095238095239, "delta_EA_greedy": 0.0, "paired_source_cluster_CI95_delta": [ 0.0, 0.0 ], "greedy_p_lower_by_position": [ 0.7020703083915643, 0.6294990547962546, 0.5201565711312489, 0.4449596093413527, 0.41064728563196784 ], "greedy_p_upper_by_position": [ 0.7020704364340432, 0.6295265726144844, 0.520198750552984, 0.4450146295475184, 0.4107120708940964 ] } } }, "context:>=32K": { "rounds": 66, "source_clusters": 1, "requests": 1, "arms": { "dyadic24-k32-t0.7": { "overlap_lower_by_position": [ 0.8183395423248736, 0.7171301187218377, 0.6086323977197038, 0.48924019235965577, 0.40781146931278045 ], "overlap_upper_by_position": [ 0.8186428812043667, 0.7175109343606068, 0.6091171900578182, 0.48974278413964656, 0.4086893634382967 ], "survival_by_position": [ 0.8063886403848366, 0.6580009528670993, 0.43898704927082743, 0.2915256896168187, 0.16256543656921563 ], "EA_by_depth": [ 0.8063886403848366, 1.4643895932519357, 1.9033766425227632, 2.194902332139582, 2.3574677687087977 ], "EA_depth5": 2.3574677687087977, "delta_EA_greedy": 0.14534655658758577, "paired_source_cluster_CI95_delta": null, "greedy_p_lower_by_position": [ 0.7712058796518665, 0.70213296983636, 0.5997583016587343, 0.47348645406493406, 0.40018051539342175 ], "greedy_p_upper_by_position": [ 0.7712142702248057, 0.7021349464239479, 0.5997612610925127, 0.4735196521048376, 0.400190268713852 ] }, "greedy": { "overlap_lower_by_position": [ 0.7712058796518665, 0.70213296983636, 0.5997583016587343, 0.47348645406493406, 0.40018051539342175 ], "overlap_upper_by_position": [ 0.7712669837407986, 0.7021555466134711, 0.5997980734810117, 0.47367706136653354, 0.400231528981128 ], "survival_by_position": [ 0.7575757575757576, 0.6212121212121212, 0.4090909090909091, 0.2727272727272727, 0.15151515151515152 ], "EA_by_depth": [ 0.7575757575757576, 1.378787878787879, 1.787878787878788, 2.0606060606060606, 2.212121212121212 ], "EA_depth5": 2.212121212121212, "delta_EA_greedy": 0.0, "paired_source_cluster_CI95_delta": null, "greedy_p_lower_by_position": [ 0.7712058796518665, 0.70213296983636, 0.5997583016587343, 0.47348645406493406, 0.40018051539342175 ], "greedy_p_upper_by_position": [ 0.7712142702248057, 0.7021349464239479, 0.5997612610925127, 0.4735196521048376, 0.400190268713852 ] } } } }, "family_deltas": { "dyadic24-k32-t0.7": [ 0.23611083212258732, 0.19390948926199458 ], "greedy": [ 0.0, 0.0 ] }, "family_uncertainty": "two held-out task families: report both; source-cluster CIs are conditional on these families", "scope": "one-round expectation on fixed target-history anchors, terminal suffix censored; not long-run throughput", "classification": "MEASURED emitted GPU q on frozen heldout anchors", "frozen_commit": "e1befc9aaa081a4f6cccbb11423a747fcd392503", "frozen_sha256": { "dev-selection.json": "72989e3c7c2bd3ab9d489d8457e02e89a1bd7fbdc1c298f52fa21715f70615be", "heldout-replay.json": "36435b607ca8c56de4ffc539c49926a50085ef4ec33071f645387c2561be0a99", "numeric.py": "c47cb3dfae376b8ffed48ca6c283a0a7dd9d1cfbc5d1b6e1a91b45aa8b28dcb2", "heldout-summary.json": "e95d0b35095db74b6886a2c52e84589391fa87144eb3e8f61414dbbe88311197", "summarize.py": "070ff608f389b107386f1020731a169d7c5785e2b01adac922b4109dae5f62e7" }, "comparisons": { "all": { "sampled_retained_tokens_per_round": 3.3055150313521673, "greedy_retained_tokens_per_round": 3.091812865497076, "delta_retained_tokens_per_round": 0.2137021658550915, "CI95_delta": [ 0.17537044298278803, 0.25443506996441134 ], "PP2_delta": 0.21370155282867476, "PP2_CI95_delta": [ 0.17536962538770962, 0.2544344155783623 ], "change_from_PP2": 6.130264167403254e-07, "PP2_greedy_retained_tokens_per_round": 3.091812865497076, "greedy_change_from_PP2": 0.0, "CI95_greedy_change_from_PP2": [ 0.0, 0.0 ], "sampled_delta_frozen_greedy": 0.2137021658550915, "CI95_sampled_delta_frozen_greedy": [ 0.17537044298278803, 0.25443506996441134 ], "point_within_PP2_interval": true }, "family:chronology": { "sampled_retained_tokens_per_round": 3.369527290975455, "greedy_retained_tokens_per_round": 3.1334164588528677, "delta_retained_tokens_per_round": 0.23611083212258732, "CI95_delta": [ 0.16316321679521495, 0.3015778018965609 ], "PP2_delta": 0.23611027063404277, "PP2_CI95_delta": [ 0.16316342853479984, 0.30157807241611817 ], "change_from_PP2": 5.614885445481832e-07, "PP2_greedy_retained_tokens_per_round": 3.1334164588528677, "greedy_change_from_PP2": 0.0, "CI95_greedy_change_from_PP2": [ 0.0, 0.0 ], "sampled_delta_frozen_greedy": 0.23611083212258732, "CI95_sampled_delta_frozen_greedy": [ 0.16316321679521495, 0.3015778018965609 ], "point_within_PP2_interval": true }, "family:motion_outline": { "sampled_retained_tokens_per_round": 3.2489755685571486, "greedy_retained_tokens_per_round": 3.055066079295154, "delta_retained_tokens_per_round": 0.19390948926199458, "CI95_delta": [ 0.1621883615015689, 0.23309372869574377 ], "PP2_delta": 0.19390883071424136, "PP2_CI95_delta": [ 0.16218746176069568, 0.23309337320225615 ], "change_from_PP2": 6.585477532183859e-07, "PP2_greedy_retained_tokens_per_round": 3.055066079295154, "greedy_change_from_PP2": 0.0, "CI95_greedy_change_from_PP2": [ 0.0, 0.0 ], "sampled_delta_frozen_greedy": 0.19390948926199458, "CI95_sampled_delta_frozen_greedy": [ 0.1621883615015689, 0.23309372869574377 ], "point_within_PP2_interval": true }, "context:16-32K": { "sampled_retained_tokens_per_round": 3.3763763183518276, "greedy_retained_tokens_per_round": 3.2238532110091747, "delta_retained_tokens_per_round": 0.15252310734265295, "CI95_delta": [ 0.10383010893025522, 0.2005272063053255 ], "PP2_delta": 0.15252232221234863, "PP2_CI95_delta": [ 0.10382912743606565, 0.20052669076899177 ], "change_from_PP2": 7.851303043260316e-07, "PP2_greedy_retained_tokens_per_round": 3.2238532110091747, "greedy_change_from_PP2": 0.0, "CI95_greedy_change_from_PP2": [ 0.0, 0.0 ], "sampled_delta_frozen_greedy": 0.15252310734265295, "CI95_sampled_delta_frozen_greedy": [ 0.10383010893025522, 0.2005272063053255 ], "point_within_PP2_interval": true }, "context:8-16K": { "sampled_retained_tokens_per_round": 3.3196860378168047, "greedy_retained_tokens_per_round": 3.0780559646539025, "delta_retained_tokens_per_round": 0.2416300731629022, "CI95_delta": [ 0.18166123017728725, 0.31049722131817714 ], "PP2_delta": 0.24162934535999314, "PP2_CI95_delta": [ 0.1816603668086756, 0.3104964738295701 ], "change_from_PP2": 7.278029090507232e-07, "PP2_greedy_retained_tokens_per_round": 3.0780559646539025, "greedy_change_from_PP2": 0.0, "CI95_greedy_change_from_PP2": [ 0.0, 0.0 ], "sampled_delta_frozen_greedy": 0.2416300731629022, "CI95_sampled_delta_frozen_greedy": [ 0.18166123017728725, 0.31049722131817714 ], "point_within_PP2_interval": true }, "context:<8K": { "sampled_retained_tokens_per_round": 3.1824902802334973, "greedy_retained_tokens_per_round": 2.923809523809524, "delta_retained_tokens_per_round": 0.2586807564239735, "CI95_delta": [ 0.16197123657259327, 0.36856276040941555 ], "PP2_delta": 0.2586805596021795, "PP2_CI95_delta": [ 0.16197031912005402, 0.36856286780395014 ], "change_from_PP2": 1.9682179397406685e-07, "PP2_greedy_retained_tokens_per_round": 2.923809523809524, "greedy_change_from_PP2": 0.0, "CI95_greedy_change_from_PP2": [ 0.0, 0.0 ], "sampled_delta_frozen_greedy": 0.2586807564239735, "CI95_sampled_delta_frozen_greedy": [ 0.16197123657259327, 0.36856276040941555 ], "point_within_PP2_interval": true }, "context:>=32K": { "sampled_retained_tokens_per_round": 3.3574677687087977, "greedy_retained_tokens_per_round": 3.212121212121212, "delta_retained_tokens_per_round": 0.14534655658758577, "CI95_delta": null, "PP2_delta": 0.14534589695382039, "PP2_CI95_delta": null, "change_from_PP2": 6.596337653874684e-07, "PP2_greedy_retained_tokens_per_round": 3.212121212121212, "greedy_change_from_PP2": 0.0, "CI95_greedy_change_from_PP2": null, "sampled_delta_frozen_greedy": 0.14534655658758577, "CI95_sampled_delta_frozen_greedy": null, "point_within_PP2_interval": null } }, "greedy_baseline_drift": { "changed_groups": 0, "group_prefix_total_differences": {}, "status": "AGGREGATES_IDENTICAL", "scope": "source/context aggregate totals, not per-anchor identity; same inputs; CPython3.13/Torch2.12 versus PP2 CPython3.12/Torch2.10" }, "input_sha256": "cd18893e5d370d6483247621acf22fe9a56f12c872a531e5d4fa447b37d1ab57" }