{ "all_passed": true, "checkpoints": { "grounded_cut_s821": "cf7099132e28e2c8789bcee308de3a6bf7ae8f428d00cd03b6e2c9aff2fb0dda", "grounded_cut_s822": "d87765232da523d8127bd53895ad485942c1bd1ee12c17205fa41519af1946f7", "grounded_cut_s823": "ca95bfca612b3e07ff0ea13a5112b9d5add1178bff88fc4f0baf8197ddb26746", "grounded_future_s821": "82e6ade42e299b16cb3088974429fcac1c28da125a959b2d3a0ed9ac9ab6e571", "grounded_future_s822": "d6b28b958d38cb0e6ede657816627180658dcd9e119da7ed887c4de470ccc9fc", "grounded_future_s823": "e99e4518412442a35dd74d64883051086eda56f97d22ae3a33c5a330c50d7ced", "numeric_future_s821": "43edca0e292e714aeeceab2b21bc1a40f33084fbfb6fd89e3c7c6dc901ec482b", "numeric_future_s822": "f3497059dda69a72e63e2dbd35b84fe377a8aaa762825e275b41aea9f70d0da3", "numeric_future_s823": "3de69de35bfb8d3c715adb9d90ac1b25e56ad1458cfeb54b255b76fc62cace45" }, "effects": { "821": { "grounding": { "delta": 0.9385, "paired_95_ci": [ 0.928, 0.9485 ] }, "ordinary_families": { "0": { "accuracy": 1.0, "lifetime_bootstrap_95_ci": [ 1.0, 1.0 ], "n": 13814, "paired_vs_numeric_95_ci": [ 0.0, 0.0 ] }, "1": { "accuracy": 1.0, "lifetime_bootstrap_95_ci": [ 1.0, 1.0 ], "n": 7218, "paired_vs_numeric_95_ci": [ 0.43684768548703923, 0.46582393882626855 ] }, "10": { "accuracy": 0.805360561582642, "lifetime_bootstrap_95_ci": [ 0.797574747836381, 0.8133098737234353 ], "n": 10969, "paired_vs_numeric_95_ci": [ 0.0327356555760956, 0.047053449063499314 ] }, "2": { "accuracy": 0.7300678354330015, "lifetime_bootstrap_95_ci": [ 0.7244758130953999, 0.7355560693070885 ], "n": 22702, "paired_vs_numeric_95_ci": [ 0.023215525231701117, 0.031436994892015714 ] }, "3": { "accuracy": 0.6404595225273739, "lifetime_bootstrap_95_ci": [ 0.6249988827314981, 0.6556016962542819 ], "n": 5571, "paired_vs_numeric_95_ci": [ -0.006816757012024072, 0.00874651947199673 ] }, "4": { "accuracy": 0.8765338256053861, "lifetime_bootstrap_95_ci": [ 0.8700413835175018, 0.8829439102713796 ], "n": 9209, "paired_vs_numeric_95_ci": [ 0.04324237548388615, 0.05280316922508261 ] }, "5": { "accuracy": 0.8566287439052704, "lifetime_bootstrap_95_ci": [ 0.8489120893527262, 0.8640732330529396 ], "n": 8614, "paired_vs_numeric_95_ci": [ 0.0407904335359734, 0.05073727637746597 ] }, "6": { "accuracy": 0.5002173598029271, "lifetime_bootstrap_95_ci": [ 0.49057595560617867, 0.5096733815865663 ], "n": 6901, "paired_vs_numeric_95_ci": [ -0.009746308037313729, 0.016087671879848057 ] }, "7": { "accuracy": 0.4966454209996645, "lifetime_bootstrap_95_ci": [ 0.4885080599962963, 0.504528795703886 ], "n": 5962, "paired_vs_numeric_95_ci": [ -0.0023558387955821608, 0.015258407937981857 ] }, "8": { "accuracy": 0.37179992122883027, "lifetime_bootstrap_95_ci": [ 0.3529334381352303, 0.3900368673926125 ], "n": 7617, "paired_vs_numeric_95_ci": [ -0.011837588368586609, 0.022985746338649634 ] }, "9": { "accuracy": 0.5021, "lifetime_bootstrap_95_ci": [ 0.4956, 0.5080025 ], "n": 10000, "paired_vs_numeric_95_ci": [ -0.0085, 0.0079 ] } }, "relation_zero": { "delta": 0.9195, "paired_95_ci": [ 0.907, 0.9305 ] }, "second_zero": { "delta": 0.9695, "paired_95_ci": [ 0.962, 0.9765 ] }, "writer_credit": { "delta": 0.5755, "paired_95_ci": [ 0.5544875, 0.5960125 ] } }, "822": { "grounding": { "delta": 0.935, "paired_95_ci": [ 0.9235, 0.9455 ] }, "ordinary_families": { "0": { "accuracy": 1.0, "lifetime_bootstrap_95_ci": [ 1.0, 1.0 ], "n": 13814, "paired_vs_numeric_95_ci": [ 0.0, 0.0 ] }, "1": { "accuracy": 1.0, "lifetime_bootstrap_95_ci": [ 1.0, 1.0 ], "n": 7218, "paired_vs_numeric_95_ci": [ 0.4434360984477531, 0.4733765089668616 ] }, "10": { "accuracy": 0.802260917130094, "lifetime_bootstrap_95_ci": [ 0.7947425799286487, 0.8103368000162623 ], "n": 10969, "paired_vs_numeric_95_ci": [ 0.026219966146918468, 0.03961661673117638 ] }, "2": { "accuracy": 0.7294071006959739, "lifetime_bootstrap_95_ci": [ 0.723459794390412, 0.7350646793443505 ], "n": 22702, "paired_vs_numeric_95_ci": [ 0.02285196008499784, 0.031895419916613744 ] }, "3": { "accuracy": 0.6372285047567762, "lifetime_bootstrap_95_ci": [ 0.6222776710863696, 0.6523274043216719 ], "n": 5571, "paired_vs_numeric_95_ci": [ -0.006134997005036631, 0.008792549770449106 ] }, "4": { "accuracy": 0.8743620371375828, "lifetime_bootstrap_95_ci": [ 0.8678399987848134, 0.8807406356456956 ], "n": 9209, "paired_vs_numeric_95_ci": [ 0.041753407677607796, 0.05159216572078535 ] }, "5": { "accuracy": 0.8540747620153238, "lifetime_bootstrap_95_ci": [ 0.846622423894907, 0.861738713735488 ], "n": 8614, "paired_vs_numeric_95_ci": [ 0.03385819983419254, 0.04471857679255813 ] }, "6": { "accuracy": 0.5010867990146356, "lifetime_bootstrap_95_ci": [ 0.49236318208077307, 0.5101722897012719 ], "n": 6901, "paired_vs_numeric_95_ci": [ -0.0101243853051779, 0.01732245853909791 ] }, "7": { "accuracy": 0.5008386447500839, "lifetime_bootstrap_95_ci": [ 0.4934628471098047, 0.5082260210614573 ], "n": 5962, "paired_vs_numeric_95_ci": [ -0.010734832973158075, 0.003860436883144213 ] }, "8": { "accuracy": 0.36996192726795324, "lifetime_bootstrap_95_ci": [ 0.35252967222454046, 0.3874115279276518 ], "n": 7617, "paired_vs_numeric_95_ci": [ -0.02302871790490305, 0.014112435891901467 ] }, "9": { "accuracy": 0.4981, "lifetime_bootstrap_95_ci": [ 0.492, 0.5039025 ], "n": 10000, "paired_vs_numeric_95_ci": [ -0.0085025, 0.008302499999999987 ] } }, "relation_zero": { "delta": 0.9195, "paired_95_ci": [ 0.907, 0.9305 ] }, "second_zero": { "delta": 0.9755, "paired_95_ci": [ 0.9685, 0.982 ] }, "writer_credit": { "delta": 0.5825, "paired_95_ci": [ 0.5609875000000001, 0.605 ] } }, "823": { "grounding": { "delta": 0.94, "paired_95_ci": [ 0.929, 0.95 ] }, "ordinary_families": { "0": { "accuracy": 1.0, "lifetime_bootstrap_95_ci": [ 1.0, 1.0 ], "n": 13814, "paired_vs_numeric_95_ci": [ 0.0, 0.0005800916188892 ] }, "1": { "accuracy": 1.0, "lifetime_bootstrap_95_ci": [ 1.0, 1.0 ], "n": 7218, "paired_vs_numeric_95_ci": [ 0.43410743467236745, 0.46507646871810315 ] }, "10": { "accuracy": 0.8075485459020877, "lifetime_bootstrap_95_ci": [ 0.8001993797602215, 0.8151366206940472 ], "n": 10969, "paired_vs_numeric_95_ci": [ 0.0338572621723962, 0.048068295626372014 ] }, "2": { "accuracy": 0.7291868557836314, "lifetime_bootstrap_95_ci": [ 0.7234498652453343, 0.734846106044913 ], "n": 22702, "paired_vs_numeric_95_ci": [ 0.02381144370745526, 0.032737013920597205 ] }, "3": { "accuracy": 0.6476395620175911, "lifetime_bootstrap_95_ci": [ 0.6330665582655827, 0.6619872931717187 ], "n": 5571, "paired_vs_numeric_95_ci": [ 0.0005371338805813189, 0.014674499134543967 ] }, "4": { "accuracy": 0.8653491149961994, "lifetime_bootstrap_95_ci": [ 0.8587660044757502, 0.8717810670262505 ], "n": 9209, "paired_vs_numeric_95_ci": [ 0.03188806772536158, 0.04021232535857154 ] }, "5": { "accuracy": 0.8479219874622708, "lifetime_bootstrap_95_ci": [ 0.8402263453372459, 0.8557715296003527 ], "n": 8614, "paired_vs_numeric_95_ci": [ 0.02999628083505561, 0.03989356097506071 ] }, "6": { "accuracy": 0.5041298362556151, "lifetime_bootstrap_95_ci": [ 0.49642900912403853, 0.5123931004493405 ], "n": 6901, "paired_vs_numeric_95_ci": [ -0.00403112606002062, 0.001596198660039464 ] }, "7": { "accuracy": 0.5010063737001006, "lifetime_bootstrap_95_ci": [ 0.49379595380270885, 0.5085485845487169 ], "n": 5962, "paired_vs_numeric_95_ci": [ -0.004024819721616636, 0.003861895865647299 ] }, "8": { "accuracy": 0.4018642510174609, "lifetime_bootstrap_95_ci": [ 0.38319438731499506, 0.4214750692655941 ], "n": 7617, "paired_vs_numeric_95_ci": [ -0.0006545380763896737, 0.030161156756118964 ] }, "9": { "accuracy": 0.5006, "lifetime_bootstrap_95_ci": [ 0.4962, 0.505 ], "n": 10000, "paired_vs_numeric_95_ci": [ -0.0047025, 0.003 ] } }, "relation_zero": { "delta": 0.9195, "paired_95_ci": [ 0.907, 0.9305 ] }, "second_zero": { "delta": 0.9755, "paired_95_ci": [ 0.9685, 0.982 ] }, "writer_credit": { "delta": 0.619, "paired_95_ci": [ 0.598, 0.641 ] } } }, "seeds": { "821": { "all_passed": true, "gates": { "challenge_drop_relation_zero": true, "challenge_drop_second_zero": true, "challenge_identity_0": true, "challenge_identity_128": true, "challenge_identity_512": true, "challenge_n": true, "challenge_onehop_0": true, "challenge_onehop_128": true, "challenge_onehop_512": true, "challenge_oracle": true, "challenge_permutation_agrees": true, "challenge_relation_cf_answer": true, "challenge_relation_cf_identity": true, "challenge_replay_agrees": true, "challenge_restore_agrees": true, "challenge_twohop_0": true, "challenge_twohop_128": true, "challenge_twohop_512": true, "challenge_twohop_ci_0": true, "challenge_twohop_ci_128": true, "challenge_twohop_ci_512": true, "challenge_value_cf_answer": true, "current_only": true, "family_10_power": true, "family_10_preserved": true, "family_2_power": true, "family_2_preserved": true, "family_3_power": true, "family_3_preserved": true, "family_4_power": true, "family_4_preserved": true, "family_5_power": true, "family_5_preserved": true, "family_6_power": true, "family_6_preserved": true, "family_7_power": true, "family_7_preserved": true, "family_8_power": true, "family_8_preserved": true, "family_9_power": true, "family_9_preserved": true, "identity_advantage_grounded_cut": true, "identity_advantage_numeric_future": true, "identity_retention": true, "long_replay": true, "long_restore": true, "ordinary_n": true, "pns_current_margin": true, "pns_drop_relation_zero": true, "pns_drop_second_zero": true, "pns_identity": true, "pns_latest": true, "pns_long_acc": true, "pns_long_delay": true, "pns_long_power": true, "pns_oracle": true, "pns_replay": true, "pns_restore": true, "pns_twohop": true, "twohop_retention": true, "unrelated_changed": true }, "unmeasured": [] }, "822": { "all_passed": true, "gates": { "challenge_drop_relation_zero": true, "challenge_drop_second_zero": true, "challenge_identity_0": true, "challenge_identity_128": true, "challenge_identity_512": true, "challenge_n": true, "challenge_onehop_0": true, "challenge_onehop_128": true, "challenge_onehop_512": true, "challenge_oracle": true, "challenge_permutation_agrees": true, "challenge_relation_cf_answer": true, "challenge_relation_cf_identity": true, "challenge_replay_agrees": true, "challenge_restore_agrees": true, "challenge_twohop_0": true, "challenge_twohop_128": true, "challenge_twohop_512": true, "challenge_twohop_ci_0": true, "challenge_twohop_ci_128": true, "challenge_twohop_ci_512": true, "challenge_value_cf_answer": true, "current_only": true, "family_10_power": true, "family_10_preserved": true, "family_2_power": true, "family_2_preserved": true, "family_3_power": true, "family_3_preserved": true, "family_4_power": true, "family_4_preserved": true, "family_5_power": true, "family_5_preserved": true, "family_6_power": true, "family_6_preserved": true, "family_7_power": true, "family_7_preserved": true, "family_8_power": true, "family_8_preserved": true, "family_9_power": true, "family_9_preserved": true, "identity_advantage_grounded_cut": true, "identity_advantage_numeric_future": true, "identity_retention": true, "long_replay": true, "long_restore": true, "ordinary_n": true, "pns_current_margin": true, "pns_drop_relation_zero": true, "pns_drop_second_zero": true, "pns_identity": true, "pns_latest": true, "pns_long_acc": true, "pns_long_delay": true, "pns_long_power": true, "pns_oracle": true, "pns_replay": true, "pns_restore": true, "pns_twohop": true, "twohop_retention": true, "unrelated_changed": true }, "unmeasured": [] }, "823": { "all_passed": true, "gates": { "challenge_drop_relation_zero": true, "challenge_drop_second_zero": true, "challenge_identity_0": true, "challenge_identity_128": true, "challenge_identity_512": true, "challenge_n": true, "challenge_onehop_0": true, "challenge_onehop_128": true, "challenge_onehop_512": true, "challenge_oracle": true, "challenge_permutation_agrees": true, "challenge_relation_cf_answer": true, "challenge_relation_cf_identity": true, "challenge_replay_agrees": true, "challenge_restore_agrees": true, "challenge_twohop_0": true, "challenge_twohop_128": true, "challenge_twohop_512": true, "challenge_twohop_ci_0": true, "challenge_twohop_ci_128": true, "challenge_twohop_ci_512": true, "challenge_value_cf_answer": true, "current_only": true, "family_10_power": true, "family_10_preserved": true, "family_2_power": true, "family_2_preserved": true, "family_3_power": true, "family_3_preserved": true, "family_4_power": true, "family_4_preserved": true, "family_5_power": true, "family_5_preserved": true, "family_6_power": true, "family_6_preserved": true, "family_7_power": true, "family_7_preserved": true, "family_8_power": true, "family_8_preserved": true, "family_9_power": true, "family_9_preserved": true, "identity_advantage_grounded_cut": true, "identity_advantage_numeric_future": true, "identity_retention": true, "long_replay": true, "long_restore": true, "ordinary_n": true, "pns_current_margin": true, "pns_drop_relation_zero": true, "pns_drop_second_zero": true, "pns_identity": true, "pns_latest": true, "pns_long_acc": true, "pns_long_delay": true, "pns_long_power": true, "pns_oracle": true, "pns_replay": true, "pns_restore": true, "pns_twohop": true, "twohop_retention": true, "unrelated_changed": true }, "unmeasured": [] } }, "sources": { "campaigns/e8_full/PROTOCOL.md": "d963c30b0012f713d1aa37d25384af26178008c8a9e9b99b66539c750c595ec5", "campaigns/e8_full/benchmark.py": "2875fa4f617210201fbbb18ed63bd37e6084a4c7e43b71627cd61356097fe172", "campaigns/e8_full/common.py": "5ef5ca21fe57da06cbf020c20d32c46b75ef52463abeb6e8c7375d79591c9843", "campaigns/e8_full/data.py": "276537eca7aa7cbd45cc18fc7d306289a01344af18a154c2168ea70393a70460", "campaigns/e8_full/evaluate.py": "91cd979a26384ed63be3acc36b401860bb522cefa41af6874b3b6e8daff41bf3", "campaigns/e8_full/gates.py": "df17f274184cb91b4c9023949fa896e7df662fea6d11208d2cd48aeaad249f3b", "campaigns/e8_full/model.py": "d06963e144e4536b599727891af9776ee12c13d4c4a523753655e04051c51b8c", "campaigns/e8_full/preflight.py": "41bd0f5ed3b6439ea2d12f29364c266c1f32506a46cc18f396d8d00f1d114c97", "campaigns/e8_full/run_campaign.py": "170ba0d116c8186f9223d5b0d17a0ed9f7ae7949ca7618e700a30cb472c69ae9", "campaigns/e8_full/train.py": "3fabd69993fe4523d4925556d299b8620387e8c0d5369c85a04a9c89c331e3e6", "campaigns/e8_identity_future/PREREGISTRATION_FULL.md": "7a5ac518262079cc3175f59ef12c01d25d6011e1af28efad26b3fb0f74301bf6", "campaigns/e8_identity_future/core.py": "33905ffd455385a657c066648349e0595aa165daecd61380b7e8c52b47968425", "pns/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "pns/common.py": "c86ea65aae22266b52e2349d9589115b75b6881a4a88eabb18fd122e319b4aa4", "pns/data/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "pns/data/binding.py": "b5721cbf0f14bad2522590f86214b6b541b20fff52e91b63079737b1dc3fd56e", "pns/data/pack.py": "6b26767b2173a825eb023768b105e2f83d3cdc8b97603eb190f4bb344ce8e582", "pns/data/tokenize.py": "bd550ac2a73ce7f5180c4bdcc67e782af86c657e8dec5e1687ab479840ab934a", "pns/data/view.py": "ee90a4c3e966efcc026ca6180e6e3291b4eba21a9df32c37dbd9ab9c7f36e88a", "pns/derange.py": "0791dd8a7478746271fffa85ef52f7ccf7f4f50492dc79bfcec20464a909daa9", "pns/eval/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "pns/model/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "pns/model/bind.py": "d18c84785e4456e41bb9962eaf91f97063fad87e729bdc7a284439ca66c80f8c", "pns/model/compose.py": "f825342fb28cda25028cca6b9d3ff484f6a57dc4b25de3b8c3ebaf0a7ab4370c", "pns/model/frozenhead.py": "71028c51cc4a709a50e51aa7703487c527af78550ffc7ca807ea5e2271685aeb", "pns/model/modules.py": "15938c7dcc34828e12358b90927b91aaa09fb666099d5d4c1e6b397afd8e483c", "pns/model/pnsr.py": "15700dae63b7bccb6dd32961f3f9106feb724d98ef74efdf48e8aeb611fbc7ed", "pns/model/relrep.py": "2f4be9f90c0780619f5fcf7407792f32831eb9688720877c96ff3c3280540afb", "pns/model/relwrite.py": "f040b482b96e1fb7280cfac25db65fad8cedee4d524cba007a0e8ae44bbcfcda", "pns/model/rmt.py": "d64cb291a26a757c372c8cbb51a344823cbc2c8b74947a2e4e57a99e303c3263", "pns/model/tx.py": "9c88fbab991b7a3748a662403b119037badf4f7f547c5ec7f6639523f0bcf64c", "pns/train/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "pns/train/loader.py": "df1fa3f4590bbcdd17f94984b666a6e4ceceaee9e9f92e9c588a0a9b52cbedfd", "pns/verdict.py": "d43470dc3c2a0f365775866bd2d031b6b52612cf77c6a97d40ea9bd1651f6e74", "pns/world/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "pns/world/questions.py": "1062709b7b777afd2b5a4a3e45793654737f785a58251ee64cd14b894ed7234d", "pns/world/reducer.py": "216ce037b62c4543dd4d6536b2039ed005cffcf2828efa507c800545932c41c6", "pns/world/schema.py": "45b06d3d2b3e44e4cf9c68c92213b645f0d2f94208bf7e4dd09b2ba95745098e", "pns/world/sim.py": "19efe037de29b4e835f1f2281489daf20959b554a86d6fa660e955c1dfdb2cae", "pns/world/verbalize.py": "5fe1c98bac552dbfdcb237dc128c2f5ee94e60eca4dd81a2366264b95ef3ff93", "tokenizer/tokenizer.json": "8bbb4fd7db2928b323b7cb7ba595e88130f49a717fb7a79b3bd33b4b5e248fcf" }, "split": "confirmation" }