| { | |
| "tasks": { | |
| "r7|full|chess_legal_v2": { | |
| "model": "r7", | |
| "stage": "full", | |
| "task": "chess_legal_v2", | |
| "rows": 500, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.4005, | |
| "debiased_accuracy": 0.426, | |
| "debiased_ci95": [ | |
| 0.38338257803751286, | |
| 0.4697458243684626 | |
| ], | |
| "order_consistency": 0.466, | |
| "nll_mean_probs": 1.2895886544670165, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.4155, | |
| 0.2225, | |
| 0.183, | |
| 0.179 | |
| ] | |
| }, | |
| "r7|full|gpqa_v2": { | |
| "model": "r7", | |
| "stage": "full", | |
| "task": "gpqa_v2", | |
| "rows": 198, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.3371212121212121, | |
| "debiased_accuracy": 0.3282828282828283, | |
| "debiased_ci95": [ | |
| 0.2666857032513807, | |
| 0.3964162165579131 | |
| ], | |
| "order_consistency": 0.23232323232323232, | |
| "nll_mean_probs": 1.3462829985829794, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.2032828282828283, | |
| 0.18686868686868688, | |
| 0.2777777777777778, | |
| 0.33207070707070707 | |
| ] | |
| }, | |
| "r7|full|gsm8k_mc10_v2": { | |
| "model": "r7", | |
| "stage": "full", | |
| "task": "gsm8k_mc10_v2", | |
| "rows": 1319, | |
| "rotations": 10, | |
| "k": 10, | |
| "rotation_mean_accuracy": 0.2858225928733891, | |
| "debiased_accuracy": 0.3062926459438969, | |
| "debiased_ci95": [ | |
| 0.282008848029071, | |
| 0.3317014748752748 | |
| ], | |
| "order_consistency": 0.14935557240333586, | |
| "nll_mean_probs": 1.9325159750133525, | |
| "ln_k": 2.302585092994046, | |
| "pred_position_share": [ | |
| 0.06739954510993176, | |
| 0.07816527672479151, | |
| 0.08695981804397271, | |
| 0.09833206974981046, | |
| 0.10151630022744504, | |
| 0.100303260045489, | |
| 0.10962850644427596, | |
| 0.11940864291129644, | |
| 0.11834723275208492, | |
| 0.1199393479909022 | |
| ] | |
| }, | |
| "r7|full|gsm8k_mc4_v2": { | |
| "model": "r7", | |
| "stage": "full", | |
| "task": "gsm8k_mc4_v2", | |
| "rows": 1319, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.515352539802881, | |
| "debiased_accuracy": 0.5382865807429871, | |
| "debiased_ci95": [ | |
| 0.5113101453003895, | |
| 0.5650406519337808 | |
| ], | |
| "order_consistency": 0.43290371493555724, | |
| "nll_mean_probs": 1.096663031900488, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.19143290371493557, | |
| 0.23294162244124336, | |
| 0.27369219105382864, | |
| 0.30193328278999243 | |
| ] | |
| }, | |
| "r7|options_only|chess_legal_v2": { | |
| "model": "r7", | |
| "stage": "options_only", | |
| "task": "chess_legal_v2", | |
| "rows": 500, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.294, | |
| "debiased_accuracy": 0.308, | |
| "debiased_ci95": [ | |
| 0.26912574776130455, | |
| 0.34980199902176684 | |
| ], | |
| "order_consistency": 0.11, | |
| "nll_mean_probs": 1.4122002220138676, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.154, | |
| 0.143, | |
| 0.259, | |
| 0.444 | |
| ] | |
| }, | |
| "r7|options_only|gpqa_v2": { | |
| "model": "r7", | |
| "stage": "options_only", | |
| "task": "gpqa_v2", | |
| "rows": 198, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.3068181818181818, | |
| "debiased_accuracy": 0.32323232323232326, | |
| "debiased_ci95": [ | |
| 0.2619851056969361, | |
| 0.3912080470479252 | |
| ], | |
| "order_consistency": 0.06565656565656566, | |
| "nll_mean_probs": 1.4115891851171167, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.15025252525252525, | |
| 0.16540404040404041, | |
| 0.3068181818181818, | |
| 0.37752525252525254 | |
| ] | |
| }, | |
| "r7|options_only|gsm8k_mc10_v2": { | |
| "model": "r7", | |
| "stage": "options_only", | |
| "task": "gsm8k_mc10_v2", | |
| "rows": 1319, | |
| "rotations": 5, | |
| "k": 10, | |
| "rotation_mean_accuracy": 0.10947687642153177, | |
| "debiased_accuracy": 0.10841546626231995, | |
| "debiased_ci95": [ | |
| 0.0927599612966865, | |
| 0.12634525114986767 | |
| ], | |
| "order_consistency": 0.003032600454890068, | |
| "nll_mean_probs": 2.377036472907572, | |
| "ln_k": 2.302585092994046, | |
| "pred_position_share": [ | |
| 0.011372251705837756, | |
| 0.004397270659590599, | |
| 0.014859742228961335, | |
| 0.0357846853677028, | |
| 0.047611827141774074, | |
| 0.08809704321455648, | |
| 0.1420773313115997, | |
| 0.20272934040940105, | |
| 0.18832448824867323, | |
| 0.264746019711903 | |
| ] | |
| }, | |
| "r7|options_only|gsm8k_mc4_v2": { | |
| "model": "r7", | |
| "stage": "options_only", | |
| "task": "gsm8k_mc4_v2", | |
| "rows": 1319, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.2505686125852919, | |
| "debiased_accuracy": 0.24336618650492797, | |
| "debiased_ci95": [ | |
| 0.22097520606722637, | |
| 0.2672476679175918 | |
| ], | |
| "order_consistency": 0.05458680818802123, | |
| "nll_mean_probs": 1.4384991214185934, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.24298711144806673, | |
| 0.07695223654283548, | |
| 0.22915087187263078, | |
| 0.45090978013646704 | |
| ] | |
| }, | |
| "raw|full|chess_legal_v2": { | |
| "model": "raw", | |
| "stage": "full", | |
| "task": "chess_legal_v2", | |
| "rows": 500, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.315, | |
| "debiased_accuracy": 0.438, | |
| "debiased_ci95": [ | |
| 0.39514834375489416, | |
| 0.4817970744771394 | |
| ], | |
| "order_consistency": 0.014, | |
| "nll_mean_probs": 1.2759298211025412, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.6895, | |
| 0.29, | |
| 0.0125, | |
| 0.008 | |
| ] | |
| }, | |
| "raw|full|gpqa_v2": { | |
| "model": "raw", | |
| "stage": "full", | |
| "task": "gpqa_v2", | |
| "rows": 198, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.31313131313131315, | |
| "debiased_accuracy": 0.3282828282828283, | |
| "debiased_ci95": [ | |
| 0.2666857032513807, | |
| 0.3964162165579131 | |
| ], | |
| "order_consistency": 0.10101010101010101, | |
| "nll_mean_probs": 1.554143105728764, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.10984848484848485, | |
| 0.63510101010101, | |
| 0.13383838383838384, | |
| 0.12121212121212122 | |
| ] | |
| }, | |
| "raw|full|gsm8k_mc10_v2": { | |
| "model": "raw", | |
| "stage": "full", | |
| "task": "gsm8k_mc10_v2", | |
| "rows": 1319, | |
| "rotations": 10, | |
| "k": 10, | |
| "rotation_mean_accuracy": 0.16467020470052868, | |
| "debiased_accuracy": 0.20545868081880211, | |
| "debiased_ci95": [ | |
| 0.18452439827071845, | |
| 0.22810362696739636 | |
| ], | |
| "order_consistency": 0.006065200909780136, | |
| "nll_mean_probs": 2.160294692789794, | |
| "ln_k": 2.302585092994046, | |
| "pred_position_share": [ | |
| 0.17505686125852918, | |
| 0.523502653525398, | |
| 0.05473843821076573, | |
| 0.054510993176648974, | |
| 0.013722517058377558, | |
| 0.03752843062926459, | |
| 0.019257012888551934, | |
| 0.06110689916603487, | |
| 0.030780894617134194, | |
| 0.02979529946929492 | |
| ] | |
| }, | |
| "raw|full|gsm8k_mc4_v2": { | |
| "model": "raw", | |
| "stage": "full", | |
| "task": "gsm8k_mc4_v2", | |
| "rows": 1319, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.36713419257012886, | |
| "debiased_accuracy": 0.43745261561789234, | |
| "debiased_ci95": [ | |
| 0.4109011521947761, | |
| 0.4643673473731633 | |
| ], | |
| "order_consistency": 0.06899166034874905, | |
| "nll_mean_probs": 1.2742964931019587, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.2579605761940864, | |
| 0.5869977255496588, | |
| 0.11694465504169825, | |
| 0.03809704321455648 | |
| ] | |
| }, | |
| "raw|options_only|chess_legal_v2": { | |
| "model": "raw", | |
| "stage": "options_only", | |
| "task": "chess_legal_v2", | |
| "rows": 500, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.252, | |
| "debiased_accuracy": 0.29, | |
| "debiased_ci95": [ | |
| 0.2519474195915272, | |
| 0.33125480345245706 | |
| ], | |
| "order_consistency": 0.0, | |
| "nll_mean_probs": 1.4114726259217971, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.0, | |
| 0.9815, | |
| 0.0075, | |
| 0.011 | |
| ] | |
| }, | |
| "raw|options_only|gpqa_v2": { | |
| "model": "raw", | |
| "stage": "options_only", | |
| "task": "gpqa_v2", | |
| "rows": 198, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.25757575757575757, | |
| "debiased_accuracy": 0.25757575757575757, | |
| "debiased_ci95": [ | |
| 0.20168506443256926, | |
| 0.32269411647466895 | |
| ], | |
| "order_consistency": 0.020202020202020204, | |
| "nll_mean_probs": 1.5601990009952065, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.015151515151515152, | |
| 0.8434343434343434, | |
| 0.07702020202020202, | |
| 0.06439393939393939 | |
| ] | |
| }, | |
| "raw|options_only|gsm8k_mc10_v2": { | |
| "model": "raw", | |
| "stage": "options_only", | |
| "task": "gsm8k_mc10_v2", | |
| "rows": 1319, | |
| "rotations": 5, | |
| "k": 10, | |
| "rotation_mean_accuracy": 0.10553449583017482, | |
| "debiased_accuracy": 0.10538286580742987, | |
| "debiased_ci95": [ | |
| 0.08994299481674577, | |
| 0.1231146297300721 | |
| ], | |
| "order_consistency": 0.0, | |
| "nll_mean_probs": 2.7989174437564013, | |
| "ln_k": 2.302585092994046, | |
| "pred_position_share": [ | |
| 0.0, | |
| 0.13525398028809704, | |
| 0.0037907505686125853, | |
| 0.0, | |
| 0.0012130401819560274, | |
| 0.00030326004548900684, | |
| 0.4363912054586808, | |
| 0.0018195602729340408, | |
| 0.3905989385898408, | |
| 0.03062926459438969 | |
| ] | |
| }, | |
| "raw|options_only|gsm8k_mc4_v2": { | |
| "model": "raw", | |
| "stage": "options_only", | |
| "task": "gsm8k_mc4_v2", | |
| "rows": 1319, | |
| "rotations": 4, | |
| "k": 4, | |
| "rotation_mean_accuracy": 0.24981046247156938, | |
| "debiased_accuracy": 0.24715693707354056, | |
| "debiased_ci95": [ | |
| 0.22463441088141858, | |
| 0.27114794797806996 | |
| ], | |
| "order_consistency": 0.0, | |
| "nll_mean_probs": 1.4600850953045421, | |
| "ln_k": 1.3862943611198906, | |
| "pred_position_share": [ | |
| 0.0, | |
| 0.9474981046247157, | |
| 0.041319181197877176, | |
| 0.011182714177407127 | |
| ] | |
| } | |
| }, | |
| "pairs": { | |
| "raw->r7|full|chess_legal_v2": { | |
| "rows": 500, | |
| "delta_pp": -1.2, | |
| "ci95_pp": [ | |
| -6.4, | |
| 3.8 | |
| ], | |
| "raw_only": 90, | |
| "r7_only": 84, | |
| "mcnemar_p": 0.7047680231195156 | |
| }, | |
| "raw->r7|full|gpqa_v2": { | |
| "rows": 198, | |
| "delta_pp": 0.0, | |
| "ci95_pp": [ | |
| -7.07070707070707, | |
| 7.07070707070707 | |
| ], | |
| "raw_only": 26, | |
| "r7_only": 26, | |
| "mcnemar_p": 1.0 | |
| }, | |
| "raw->r7|full|gsm8k_mc10_v2": { | |
| "rows": 1319, | |
| "delta_pp": 10.083396512509477, | |
| "ci95_pp": [ | |
| 7.050796057619409, | |
| 13.040181956027293 | |
| ], | |
| "raw_only": 150, | |
| "r7_only": 283, | |
| "mcnemar_p": 1.625471377453816e-10 | |
| }, | |
| "raw->r7|full|gsm8k_mc4_v2": { | |
| "rows": 1319, | |
| "delta_pp": 10.083396512509477, | |
| "ci95_pp": [ | |
| 6.444275966641396, | |
| 13.722517058377559 | |
| ], | |
| "raw_only": 226, | |
| "r7_only": 359, | |
| "mcnemar_p": 4.2453654446737796e-08 | |
| }, | |
| "raw->r7|options_only|chess_legal_v2": { | |
| "rows": 500, | |
| "delta_pp": 1.8, | |
| "ci95_pp": [ | |
| -3.0, | |
| 7.199999999999999 | |
| ], | |
| "raw_only": 79, | |
| "r7_only": 88, | |
| "mcnemar_p": 0.5360050623796357 | |
| }, | |
| "raw->r7|options_only|gpqa_v2": { | |
| "rows": 198, | |
| "delta_pp": 6.565656565656566, | |
| "ci95_pp": [ | |
| -1.0101010101010102, | |
| 14.14141414141414 | |
| ], | |
| "raw_only": 24, | |
| "r7_only": 37, | |
| "mcnemar_p": 0.12373144253809217 | |
| }, | |
| "raw->r7|options_only|gsm8k_mc10_v2": { | |
| "rows": 1319, | |
| "delta_pp": 0.3032600454890068, | |
| "ci95_pp": [ | |
| -2.047005307050796, | |
| 2.577710386656558 | |
| ], | |
| "raw_only": 119, | |
| "r7_only": 123, | |
| "mcnemar_p": 0.8471289384306088 | |
| }, | |
| "raw->r7|options_only|gsm8k_mc4_v2": { | |
| "rows": 1319, | |
| "delta_pp": -0.37907505686125853, | |
| "ci95_pp": [ | |
| -3.790750568612585, | |
| 2.9567854435178167 | |
| ], | |
| "raw_only": 261, | |
| "r7_only": 256, | |
| "mcnemar_p": 0.8603786002624875 | |
| } | |
| } | |
| } | |