| { |
| "base": { |
| "task_count": 500, |
| "sample_count": 32000, |
| "correct_count": 12142, |
| "correct_fraction": { |
| "mean": 0.3794375, |
| "all_task_mean": 0.3794375, |
| "ci95": [ |
| 0.34740546875, |
| 0.41365703125 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "pass_at_k": { |
| "1": { |
| "mean": 0.3794375, |
| "all_task_mean": 0.3794375, |
| "ci95": [ |
| 0.34740546875, |
| 0.41365703125 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "4": { |
| "mean": 0.5507654491198912, |
| "all_task_mean": 0.5507654491198912, |
| "ci95": [ |
| 0.5132311306218681, |
| 0.5890580618405479 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 0.6109743277142735, |
| "all_task_mean": 0.6109743277142735, |
| "ci95": [ |
| 0.5724928416444336, |
| 0.6497163629340588 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 0.6569243830645763, |
| "all_task_mean": 0.6569243830645763, |
| "ci95": [ |
| 0.6184579345767386, |
| 0.6952451294337093 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 0.72, |
| "all_task_mean": 0.72, |
| "ci95": [ |
| 0.6819500000000001, |
| 0.758 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| } |
| }, |
| "implementation_coverage_at_k": { |
| "1": { |
| "mean": 0.3794375, |
| "all_task_mean": 0.3794375, |
| "ci95": [ |
| 0.34740546875, |
| 0.41365703125 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "4": { |
| "mean": 1.2665641226612274, |
| "all_task_mean": 1.2665641226612274, |
| "ci95": [ |
| 1.159415250025182, |
| 1.3736470797606455 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 2.2670132030155092, |
| "all_task_mean": 2.2670132030155092, |
| "ci95": [ |
| 2.0738004189885935, |
| 2.4590917365037095 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 4.039637834881175, |
| "all_task_mean": 4.039637834881175, |
| "ci95": [ |
| 3.6929032849396326, |
| 4.390863679872534 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 12.802, |
| "all_task_mean": 12.802, |
| "ci95": [ |
| 11.69585, |
| 13.9681 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| } |
| }, |
| "correct_matched_coverage_at_budgets": { |
| "4": { |
| "mean": 3.3583723736089284, |
| "all_task_mean": null, |
| "ci95": [ |
| 3.282182510720265, |
| 3.439956670735758 |
| ], |
| "eligible_tasks": 322, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_4_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 5.986525949447583, |
| "all_task_mean": null, |
| "ci95": [ |
| 5.779330460447849, |
| 6.196882660452379 |
| ], |
| "eligible_tasks": 294, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_8_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 10.42331543495907, |
| "all_task_mean": null, |
| "ci95": [ |
| 9.959521542093748, |
| 10.910493297809975 |
| ], |
| "eligible_tasks": 251, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_16_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 19.875, |
| "all_task_mean": null, |
| "ci95": [ |
| 15.0, |
| 25.04092307692307 |
| ], |
| "eligible_tasks": 24, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_64_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| } |
| } |
| }, |
| "plain": { |
| "task_count": 500, |
| "sample_count": 32000, |
| "correct_count": 13263, |
| "correct_fraction": { |
| "mean": 0.41446875, |
| "all_task_mean": 0.41446875, |
| "ci95": [ |
| 0.38040546875, |
| 0.45168984375 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "pass_at_k": { |
| "1": { |
| "mean": 0.41446875, |
| "all_task_mean": 0.41446875, |
| "ci95": [ |
| 0.38040546875, |
| 0.45168984375 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "4": { |
| "mean": 0.5654111549696559, |
| "all_task_mean": 0.5654111549696559, |
| "ci95": [ |
| 0.5268074897383598, |
| 0.6045970197961521 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 0.6185723086964429, |
| "all_task_mean": 0.6185723086964429, |
| "ci95": [ |
| 0.5793926136329006, |
| 0.65815805619536 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 0.6563636760539059, |
| "all_task_mean": 0.6563636760539059, |
| "ci95": [ |
| 0.6166121006457762, |
| 0.6960772759859236 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 0.704, |
| "all_task_mean": 0.704, |
| "ci95": [ |
| 0.664, |
| 0.744 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| } |
| }, |
| "implementation_coverage_at_k": { |
| "1": { |
| "mean": 0.41446875, |
| "all_task_mean": 0.41446875, |
| "ci95": [ |
| 0.38040546875, |
| 0.45168984375 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "4": { |
| "mean": 1.1661383023595477, |
| "all_task_mean": 1.1661383023595477, |
| "ci95": [ |
| 1.070903410028078, |
| 1.2672731045868904 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 1.9228737911818574, |
| "all_task_mean": 1.9228737911818574, |
| "ci95": [ |
| 1.7571056197779187, |
| 2.101574059760503 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 3.1592155356398983, |
| "all_task_mean": 3.1592155356398983, |
| "ci95": [ |
| 2.8673068380698647, |
| 3.47591123889947 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 8.506, |
| "all_task_mean": 8.506, |
| "ci95": [ |
| 7.60195, |
| 9.46405 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| } |
| }, |
| "correct_matched_coverage_at_budgets": { |
| "4": { |
| "mean": 2.856476213813779, |
| "all_task_mean": null, |
| "ci95": [ |
| 2.7541549099452163, |
| 2.9606601396926697 |
| ], |
| "eligible_tasks": 321, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_4_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 4.690364473715527, |
| "all_task_mean": null, |
| "ci95": [ |
| 4.453332768614563, |
| 4.9366147006484855 |
| ], |
| "eligible_tasks": 305, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_8_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 7.3244981368337205, |
| "all_task_mean": null, |
| "ci95": [ |
| 6.814141127178505, |
| 7.842568973120032 |
| ], |
| "eligible_tasks": 263, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_16_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 8.686274509803921, |
| "all_task_mean": null, |
| "ci95": [ |
| 6.377226853883282, |
| 11.162976249381494 |
| ], |
| "eligible_tasks": 51, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_64_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| } |
| } |
| }, |
| "spd_hard": { |
| "task_count": 500, |
| "sample_count": 32000, |
| "correct_count": 13365, |
| "correct_fraction": { |
| "mean": 0.41765625, |
| "all_task_mean": 0.41765625, |
| "ci95": [ |
| 0.384625, |
| 0.4546578125 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "pass_at_k": { |
| "1": { |
| "mean": 0.41765625, |
| "all_task_mean": 0.41765625, |
| "ci95": [ |
| 0.384625, |
| 0.4546578125 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "4": { |
| "mean": 0.5659676600941804, |
| "all_task_mean": 0.5659676600941804, |
| "ci95": [ |
| 0.527518317657576, |
| 0.604875113633502 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 0.6187692750882325, |
| "all_task_mean": 0.6187692750882325, |
| "ci95": [ |
| 0.5796125930854173, |
| 0.6583837108361063 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 0.656461003056232, |
| "all_task_mean": 0.656461003056232, |
| "ci95": [ |
| 0.616541219686637, |
| 0.6960544516270536 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 0.702, |
| "all_task_mean": 0.702, |
| "ci95": [ |
| 0.662, |
| 0.7420499999999998 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| } |
| }, |
| "implementation_coverage_at_k": { |
| "1": { |
| "mean": 0.41765625, |
| "all_task_mean": 0.41765625, |
| "ci95": [ |
| 0.384625, |
| 0.4546578125 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "4": { |
| "mean": 1.1526910962957369, |
| "all_task_mean": 1.1526910962957369, |
| "ci95": [ |
| 1.059633787473874, |
| 1.2513327874361007 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 1.8917267799240527, |
| "all_task_mean": 1.8917267799240527, |
| "ci95": [ |
| 1.7283757935135853, |
| 2.062184461404335 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 3.103388895756195, |
| "all_task_mean": 3.103388895756195, |
| "ci95": [ |
| 2.817451519989818, |
| 3.40744240345712 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 8.388, |
| "all_task_mean": 8.388, |
| "ci95": [ |
| 7.514, |
| 9.322049999999999 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| } |
| }, |
| "correct_matched_coverage_at_budgets": { |
| "4": { |
| "mean": 2.8097969039470296, |
| "all_task_mean": null, |
| "ci95": [ |
| 2.702378107013863, |
| 2.912624204119433 |
| ], |
| "eligible_tasks": 321, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_4_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 4.589740759204707, |
| "all_task_mean": null, |
| "ci95": [ |
| 4.341571130697039, |
| 4.8420976777276685 |
| ], |
| "eligible_tasks": 303, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_8_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 7.136154971929802, |
| "all_task_mean": null, |
| "ci95": [ |
| 6.612424214688567, |
| 7.64193275562665 |
| ], |
| "eligible_tasks": 264, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_16_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 7.981132075471698, |
| "all_task_mean": null, |
| "ci95": [ |
| 5.9298245614035086, |
| 10.380074468085107 |
| ], |
| "eligible_tasks": 53, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_64_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| } |
| } |
| }, |
| "spectral_soft": { |
| "task_count": 500, |
| "sample_count": 32000, |
| "correct_count": 12906, |
| "correct_fraction": { |
| "mean": 0.4033125, |
| "all_task_mean": 0.4033125, |
| "ci95": [ |
| 0.37115312500000003, |
| 0.43831796874999995 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "pass_at_k": { |
| "1": { |
| "mean": 0.4033125, |
| "all_task_mean": 0.4033125, |
| "ci95": [ |
| 0.37115312500000003, |
| 0.43831796874999995 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "4": { |
| "mean": 0.562148161088867, |
| "all_task_mean": 0.562148161088867, |
| "ci95": [ |
| 0.5239757778858503, |
| 0.6009773591542646 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 0.6189282395695613, |
| "all_task_mean": 0.6189282395695613, |
| "ci95": [ |
| 0.5799531764909873, |
| 0.6581920368265824 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 0.6622507815182023, |
| "all_task_mean": 0.6622507815182023, |
| "ci95": [ |
| 0.6233787417205545, |
| 0.7018316378569717 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 0.726, |
| "all_task_mean": 0.726, |
| "ci95": [ |
| 0.688, |
| 0.766 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| } |
| }, |
| "implementation_coverage_at_k": { |
| "1": { |
| "mean": 0.4033125, |
| "all_task_mean": 0.4033125, |
| "ci95": [ |
| 0.37115312500000003, |
| 0.43831796874999995 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "4": { |
| "mean": 1.243749039938556, |
| "all_task_mean": 1.243749039938556, |
| "ci95": [ |
| 1.1431083542658207, |
| 1.348755115711012 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 2.161080512531813, |
| "all_task_mean": 2.161080512531813, |
| "ci95": [ |
| 1.9763838957574165, |
| 2.3468053037933396 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 3.758255772115641, |
| "all_task_mean": 3.758255772115641, |
| "ci95": [ |
| 3.4251782302115035, |
| 4.103086515045223 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 11.51, |
| "all_task_mean": 11.51, |
| "ci95": [ |
| 10.424, |
| 12.652399999999997 |
| ], |
| "eligible_tasks": 500, |
| "total_tasks": 500, |
| "population": "tasks_with_available_estimates", |
| "bootstrap_valid_replicates": 2000 |
| } |
| }, |
| "correct_matched_coverage_at_budgets": { |
| "4": { |
| "mean": 3.143291510215674, |
| "all_task_mean": null, |
| "ci95": [ |
| 3.052151432442491, |
| 3.236868415837648 |
| ], |
| "eligible_tasks": 323, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_4_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "8": { |
| "mean": 5.451019508347272, |
| "all_task_mean": null, |
| "ci95": [ |
| 5.221458862962949, |
| 5.691262607579812 |
| ], |
| "eligible_tasks": 302, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_8_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "16": { |
| "mean": 9.09058337320117, |
| "all_task_mean": null, |
| "ci95": [ |
| 8.580622492811118, |
| 9.615960487699715 |
| ], |
| "eligible_tasks": 259, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_16_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| }, |
| "64": { |
| "mean": 12.176470588235293, |
| "all_task_mean": null, |
| "ci95": [ |
| 9.37483552631579, |
| 15.0 |
| ], |
| "eligible_tasks": 34, |
| "total_tasks": 500, |
| "population": "tasks_with_complete_labels_and_at_least_64_correct_samples", |
| "bootstrap_valid_replicates": 2000 |
| } |
| } |
| } |
| } |
|
|
