| { |
| "actual_model_predictions_all_stages": 25471, |
| "benchmark_predictions_including_allocation_probe": 12012, |
| "calibration": { |
| "independent-fp32": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2925, |
| "initial_complete_functions": 154, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 400, |
| "actual_model_predictions": 1380, |
| "actual_feedback_predictions": 612, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 752, |
| "bilingual_initial_mask_disagreements": 180, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 154, |
| 112, |
| 66, |
| 52 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 154, |
| 266, |
| 332, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2925, |
| 4484, |
| 5423, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 13253190, |
| "summed_initial_probability_mass_outside_full_target": 263.40002416328724, |
| "summed_initial_full_target_nll": 525.1140375810159 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 283, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 28, |
| "actual_model_predictions": 107, |
| "actual_feedback_predictions": 43, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 68, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 283, |
| 398, |
| 454, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1025626, |
| "summed_initial_probability_mass_outside_full_target": 19.10443247631663, |
| "summed_initial_full_target_nll": 43.2229110057714 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 283, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 16, |
| "actual_model_predictions": 96, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 80, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 16, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 283, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 863960, |
| "summed_initial_probability_mass_outside_full_target": 19.687865571513182, |
| "summed_initial_full_target_nll": 45.46758740092145 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 294, |
| "initial_complete_functions": 18, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 27, |
| "actual_model_predictions": 101, |
| "actual_feedback_predictions": 37, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 69, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 18, |
| 6, |
| 3, |
| 5 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 18, |
| 24, |
| 27, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 294, |
| 390, |
| 435, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 993953, |
| "summed_initial_probability_mass_outside_full_target": 22.950683253568936, |
| "summed_initial_full_target_nll": 40.61678484472798 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 195, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 29, |
| "actual_model_predictions": 122, |
| "actual_feedback_predictions": 58, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 67, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 11, |
| 9, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 23, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 195, |
| 371, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1162706, |
| "summed_initial_probability_mass_outside_full_target": 23.849738121665013, |
| "summed_initial_full_target_nll": 44.54739746654518 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 148, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 49, |
| "actual_model_predictions": 132, |
| "actual_feedback_predictions": 68, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 47, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 7, |
| 9, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 15, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 148, |
| 265, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1238371, |
| "summed_initial_probability_mass_outside_full_target": 23.59054887476337, |
| "summed_initial_full_target_nll": 43.87110768353832 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 156, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 30, |
| "actual_model_predictions": 122, |
| "actual_feedback_predictions": 58, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 66, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 18, |
| 6, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 26, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 156, |
| 420, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1119791, |
| "summed_initial_probability_mass_outside_full_target": 23.618873349610595, |
| "summed_initial_full_target_nll": 43.69315234621296 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 353, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 31, |
| "actual_model_predictions": 109, |
| "actual_feedback_predictions": 45, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 65, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 9, |
| 5, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 23, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 353, |
| 431, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1102625, |
| "summed_initial_probability_mass_outside_full_target": 17.488844136684623, |
| "summed_initial_full_target_nll": 44.11157948999173 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 438, |
| "initial_complete_functions": 18, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 20, |
| "actual_model_predictions": 102, |
| "actual_feedback_predictions": 38, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 76, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 18, |
| 9, |
| 4, |
| 1 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 18, |
| 27, |
| 31, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 438, |
| 467, |
| 501, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1003335, |
| "summed_initial_probability_mass_outside_full_target": 17.437517033614686, |
| "summed_initial_full_target_nll": 43.31409860785471 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 170, |
| "initial_complete_functions": 10, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 44, |
| "actual_model_predictions": 124, |
| "actual_feedback_predictions": 60, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 52, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 10, |
| 8, |
| 6, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 10, |
| 18, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 170, |
| 313, |
| 399, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1234497, |
| "summed_initial_probability_mass_outside_full_target": 23.961364568377736, |
| "summed_initial_full_target_nll": 44.23390374255849 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 248, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 39, |
| "actual_model_predictions": 116, |
| "actual_feedback_predictions": 52, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 57, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 4, |
| 7, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 18, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 248, |
| 303, |
| 409, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1096376, |
| "summed_initial_probability_mass_outside_full_target": 23.898001494184285, |
| "summed_initial_full_target_nll": 43.978782570753594 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 145, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 131, |
| "actual_feedback_predictions": 67, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 145, |
| 273, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1235623, |
| "summed_initial_probability_mass_outside_full_target": 23.993335421515553, |
| "summed_initial_full_target_nll": 44.35708308656053 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 212, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 39, |
| "actual_model_predictions": 118, |
| "actual_feedback_predictions": 54, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 57, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 5, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 212, |
| 341, |
| 415, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1176327, |
| "summed_initial_probability_mass_outside_full_target": 23.818819861472818, |
| "summed_initial_full_target_nll": 43.69964933557976 |
| } |
| } |
| }, |
| "independent-ptq_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2755, |
| "initial_complete_functions": 144, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 438, |
| "actual_model_predictions": 1418, |
| "actual_feedback_predictions": 650, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 714, |
| "bilingual_initial_mask_disagreements": 168, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 144, |
| 95, |
| 92, |
| 53 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 144, |
| 239, |
| 331, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2755, |
| 4159, |
| 5402, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 15562753, |
| "summed_initial_probability_mass_outside_full_target": 264.72765764348514, |
| "summed_initial_full_target_nll": 525.8331452424557 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 283, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 24, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 72, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 283, |
| 399, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1078792, |
| "summed_initial_probability_mass_outside_full_target": 19.27240256645411, |
| "summed_initial_full_target_nll": 43.70968475713822 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 283, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 23, |
| "actual_model_predictions": 110, |
| "actual_feedback_predictions": 46, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 73, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 9, |
| 7, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 25, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 283, |
| 413, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1181168, |
| "summed_initial_probability_mass_outside_full_target": 19.90529919010544, |
| "summed_initial_full_target_nll": 44.27159617898773 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 133, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 44, |
| "actual_model_predictions": 124, |
| "actual_feedback_predictions": 60, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 52, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 12, |
| 4, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 20, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 133, |
| 327, |
| 389, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1554212, |
| "summed_initial_probability_mass_outside_full_target": 23.235460226126936, |
| "summed_initial_full_target_nll": 41.68152078748247 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 196, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 12, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 196, |
| 326, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1280623, |
| "summed_initial_probability_mass_outside_full_target": 23.941642513172376, |
| "summed_initial_full_target_nll": 44.344627617161045 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 148, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 148, |
| 274, |
| 397, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1351458, |
| "summed_initial_probability_mass_outside_full_target": 23.650489271657296, |
| "summed_initial_full_target_nll": 44.06002748528022 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 214, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 37, |
| "actual_model_predictions": 122, |
| "actual_feedback_predictions": 58, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 59, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 7, |
| 9, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 19, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 214, |
| 320, |
| 454, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1312297, |
| "summed_initial_probability_mass_outside_full_target": 23.76501813433186, |
| "summed_initial_full_target_nll": 43.53052557066708 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 353, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 114, |
| "actual_feedback_predictions": 50, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 4, |
| 10, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 18, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 353, |
| 438, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1296044, |
| "summed_initial_probability_mass_outside_full_target": 17.572842196378826, |
| "summed_initial_full_target_nll": 44.26594206199383 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 438, |
| "initial_complete_functions": 18, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 18, |
| "actual_model_predictions": 100, |
| "actual_feedback_predictions": 36, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 78, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 18, |
| 10, |
| 4, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 18, |
| 28, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 438, |
| 478, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1063375, |
| "summed_initial_probability_mass_outside_full_target": 17.55521231734092, |
| "summed_initial_full_target_nll": 42.98990900789376 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 140, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 9, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 140, |
| 283, |
| 411, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1386664, |
| "summed_initial_probability_mass_outside_full_target": 23.977058211833864, |
| "summed_initial_full_target_nll": 44.33217621037191 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 205, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 42, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 54, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 6, |
| 6, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 18, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 205, |
| 309, |
| 396, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1299754, |
| "summed_initial_probability_mass_outside_full_target": 23.94919347170267, |
| "summed_initial_full_target_nll": 44.1872961286787 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 145, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 130, |
| "actual_feedback_predictions": 66, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 145, |
| 272, |
| 400, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1414032, |
| "summed_initial_probability_mass_outside_full_target": 23.983555643931894, |
| "summed_initial_full_target_nll": 44.3405321750111 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 217, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 39, |
| "actual_model_predictions": 118, |
| "actual_feedback_predictions": 54, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 57, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 7, |
| 7, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 19, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 217, |
| 320, |
| 429, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1344334, |
| "summed_initial_probability_mass_outside_full_target": 23.919483900448835, |
| "summed_initial_full_target_nll": 44.11930726178957 |
| } |
| } |
| }, |
| "independent-qat_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2833, |
| "initial_complete_functions": 148, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 415, |
| "actual_model_predictions": 1397, |
| "actual_feedback_predictions": 629, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 737, |
| "bilingual_initial_mask_disagreements": 166, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 148, |
| 114, |
| 65, |
| 57 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 148, |
| 262, |
| 327, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2833, |
| 4382, |
| 5337, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 15247841, |
| "summed_initial_probability_mass_outside_full_target": 263.43281332204407, |
| "summed_initial_full_target_nll": 522.2127398246924 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 283, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 31, |
| "actual_model_predictions": 110, |
| "actual_feedback_predictions": 46, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 65, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 7, |
| 3, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 23, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 283, |
| 382, |
| 424, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1235583, |
| "summed_initial_probability_mass_outside_full_target": 19.12043977919033, |
| "summed_initial_full_target_nll": 42.83170988137251 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 283, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 16, |
| "actual_model_predictions": 96, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 80, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 16, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 283, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1048411, |
| "summed_initial_probability_mass_outside_full_target": 19.753837449680002, |
| "summed_initial_full_target_nll": 44.932851107514765 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 198, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 115, |
| "actual_feedback_predictions": 51, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 10, |
| 4, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 22, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 198, |
| 357, |
| 420, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1239832, |
| "summed_initial_probability_mass_outside_full_target": 22.90731458391581, |
| "summed_initial_full_target_nll": 40.47629878516883 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 257, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 25, |
| "actual_model_predictions": 114, |
| "actual_feedback_predictions": 50, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 71, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 7, |
| 9, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 23, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 257, |
| 371, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1288706, |
| "summed_initial_probability_mass_outside_full_target": 23.77594026695118, |
| "summed_initial_full_target_nll": 44.11102196754362 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 148, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 52, |
| "actual_model_predictions": 133, |
| "actual_feedback_predictions": 69, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 44, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 7, |
| 6, |
| 11 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 15, |
| 21, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 148, |
| 266, |
| 356, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1420457, |
| "summed_initial_probability_mass_outside_full_target": 23.637616891284992, |
| "summed_initial_full_target_nll": 43.59599042773189 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 214, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 28, |
| "actual_model_predictions": 114, |
| "actual_feedback_predictions": 50, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 68, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 14, |
| 4, |
| 2 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 26, |
| 30, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 214, |
| 422, |
| 481, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1216253, |
| "summed_initial_probability_mass_outside_full_target": 23.642319032168224, |
| "summed_initial_full_target_nll": 43.46101273353078 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 352, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 26, |
| "actual_model_predictions": 107, |
| "actual_feedback_predictions": 43, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 70, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 12, |
| 4, |
| 2 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 26, |
| 30, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 352, |
| 435, |
| 493, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1149702, |
| "summed_initial_probability_mass_outside_full_target": 17.546220340549805, |
| "summed_initial_full_target_nll": 43.79877485824531 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 438, |
| "initial_complete_functions": 18, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 19, |
| "actual_model_predictions": 100, |
| "actual_feedback_predictions": 36, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 77, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 18, |
| 10, |
| 3, |
| 1 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 18, |
| 28, |
| 31, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 438, |
| 478, |
| 506, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1062255, |
| "summed_initial_probability_mass_outside_full_target": 17.380628072727674, |
| "summed_initial_full_target_nll": 42.72775317225145 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 140, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 127, |
| "actual_feedback_predictions": 63, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 8, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 140, |
| 298, |
| 410, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1369659, |
| "summed_initial_probability_mass_outside_full_target": 23.975692659184034, |
| "summed_initial_full_target_nll": 44.29112537198986 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 246, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 39, |
| "actual_model_predictions": 116, |
| "actual_feedback_predictions": 52, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 57, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 4, |
| 7, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 18, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 246, |
| 303, |
| 409, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1326789, |
| "summed_initial_probability_mass_outside_full_target": 23.881407951430596, |
| "summed_initial_full_target_nll": 43.907869135967424 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 145, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 145, |
| 290, |
| 400, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1393536, |
| "summed_initial_probability_mass_outside_full_target": 24.000471580105337, |
| "summed_initial_full_target_nll": 44.40211283864889 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 129, |
| "initial_complete_functions": 6, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 50, |
| "actual_model_predictions": 137, |
| "actual_feedback_predictions": 73, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 46, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 6, |
| 9, |
| 10, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 6, |
| 15, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 129, |
| 268, |
| 414, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1496658, |
| "summed_initial_probability_mass_outside_full_target": 23.810924714855535, |
| "summed_initial_full_target_nll": 43.676219544727466 |
| } |
| } |
| }, |
| "joint-fp32": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2742, |
| "initial_complete_functions": 144, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 435, |
| "actual_model_predictions": 754, |
| "actual_feedback_predictions": 370, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 717, |
| "bilingual_initial_mask_disagreements": 174, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 144, |
| 110, |
| 65, |
| 65 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 144, |
| 254, |
| 319, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2742, |
| 4250, |
| 5266, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 9034301, |
| "summed_initial_probability_mass_outside_full_target": 265.097424402833, |
| "summed_initial_full_target_nll": 525.706839299999 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 454, |
| "initial_complete_functions": 28, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 4, |
| "actual_model_predictions": 36, |
| "actual_feedback_predictions": 4, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 92, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 28, |
| 4, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 28, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 454, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 397626, |
| "summed_initial_probability_mass_outside_full_target": 19.295918405056, |
| "summed_initial_full_target_nll": 42.05230999828956 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 283, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 24, |
| "actual_model_predictions": 56, |
| "actual_feedback_predictions": 24, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 72, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 283, |
| 399, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 691996, |
| "summed_initial_probability_mass_outside_full_target": 19.75831100344658, |
| "summed_initial_full_target_nll": 44.354622241116715 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 197, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 61, |
| "actual_feedback_predictions": 29, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 11, |
| 6, |
| 3 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 23, |
| 29, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 197, |
| 372, |
| 466, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 703751, |
| "summed_initial_probability_mass_outside_full_target": 22.962988913059235, |
| "summed_initial_full_target_nll": 40.60489804292393 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 135, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 34, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 62, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 6, |
| 2 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 30, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 135, |
| 387, |
| 481, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 773042, |
| "summed_initial_probability_mass_outside_full_target": 23.989022821187973, |
| "summed_initial_full_target_nll": 45.035760070065955 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 148, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 148, |
| 278, |
| 396, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 860210, |
| "summed_initial_probability_mass_outside_full_target": 23.692762196063995, |
| "summed_initial_full_target_nll": 43.36107756314573 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 156, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 71, |
| "actual_feedback_predictions": 39, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 156, |
| 289, |
| 396, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 981792, |
| "summed_initial_probability_mass_outside_full_target": 23.94161143898964, |
| "summed_initial_full_target_nll": 44.776263126191346 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 323, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 60, |
| "actual_feedback_predictions": 28, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 323, |
| 401, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 693500, |
| "summed_initial_probability_mass_outside_full_target": 17.63266098499298, |
| "summed_initial_full_target_nll": 43.67686643914 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 438, |
| "initial_complete_functions": 18, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 22, |
| "actual_model_predictions": 50, |
| "actual_feedback_predictions": 18, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 74, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 18, |
| 10, |
| 0, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 18, |
| 28, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 438, |
| 478, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 592501, |
| "summed_initial_probability_mass_outside_full_target": 17.861520037055016, |
| "summed_initial_full_target_nll": 44.442449537424736 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 138, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 49, |
| "actual_model_predictions": 73, |
| "actual_feedback_predictions": 41, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 47, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 7, |
| 9, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 15, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 138, |
| 245, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 863796, |
| "summed_initial_probability_mass_outside_full_target": 23.99233341217041, |
| "summed_initial_full_target_nll": 44.35710612136575 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 145, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 50, |
| "actual_model_predictions": 75, |
| "actual_feedback_predictions": 43, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 46, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 5, |
| 12, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 13, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 145, |
| 238, |
| 413, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 905173, |
| "summed_initial_probability_mass_outside_full_target": 23.991549998521805, |
| "summed_initial_full_target_nll": 44.39267490885772 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 178, |
| "initial_complete_functions": 10, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 67, |
| "actual_feedback_predictions": 35, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 10, |
| 9, |
| 1, |
| 12 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 10, |
| 19, |
| 20, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 178, |
| 327, |
| 345, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 770126, |
| "summed_initial_probability_mass_outside_full_target": 24.004550874233246, |
| "summed_initial_full_target_nll": 44.387888447202634 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 147, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 69, |
| "actual_feedback_predictions": 37, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 11, |
| 4, |
| 9 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 19, |
| 23, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 147, |
| 324, |
| 388, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 800788, |
| "summed_initial_probability_mass_outside_full_target": 23.974194318056107, |
| "summed_initial_full_target_nll": 44.2649228042747 |
| } |
| } |
| }, |
| "joint-ptq_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2708, |
| "initial_complete_functions": 145, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 473, |
| "actual_model_predictions": 779, |
| "actual_feedback_predictions": 395, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 679, |
| "bilingual_initial_mask_disagreements": 44, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 145, |
| 83, |
| 78, |
| 78 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 145, |
| 228, |
| 306, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2708, |
| 3960, |
| 5066, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 9318991, |
| "summed_initial_probability_mass_outside_full_target": 266.2105183303356, |
| "summed_initial_full_target_nll": 530.358279973884 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 311, |
| "initial_complete_functions": 18, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 34, |
| "actual_model_predictions": 58, |
| "actual_feedback_predictions": 26, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 62, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 18, |
| 2, |
| 4, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 18, |
| 20, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 311, |
| 339, |
| 396, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 682167, |
| "summed_initial_probability_mass_outside_full_target": 19.47901126742363, |
| "summed_initial_full_target_nll": 43.485937632730774 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 283, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 33, |
| "actual_model_predictions": 62, |
| "actual_feedback_predictions": 30, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 63, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 2, |
| 11, |
| 3 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 18, |
| 29, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 283, |
| 311, |
| 468, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 727585, |
| "summed_initial_probability_mass_outside_full_target": 19.622833400964737, |
| "summed_initial_full_target_nll": 44.30568677905417 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 261, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 39, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 57, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 0, |
| 9, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 16, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 261, |
| 262, |
| 406, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 830913, |
| "summed_initial_probability_mass_outside_full_target": 23.538529247045517, |
| "summed_initial_full_target_nll": 42.73027621232325 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 135, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 68, |
| "actual_feedback_predictions": 36, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 12, |
| 12, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 20, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 135, |
| 325, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 802709, |
| "summed_initial_probability_mass_outside_full_target": 23.978688687086105, |
| "summed_initial_full_target_nll": 44.63129659102326 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 186, |
| "initial_complete_functions": 10, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 50, |
| "actual_model_predictions": 70, |
| "actual_feedback_predictions": 38, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 46, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 10, |
| 6, |
| 4, |
| 12 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 10, |
| 16, |
| 20, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 186, |
| 280, |
| 341, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 844709, |
| "summed_initial_probability_mass_outside_full_target": 23.85689190030098, |
| "summed_initial_full_target_nll": 43.95067680696744 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 156, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 41, |
| "actual_model_predictions": 65, |
| "actual_feedback_predictions": 33, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 55, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 15, |
| 1, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 23, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 156, |
| 378, |
| 396, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 763746, |
| "summed_initial_probability_mass_outside_full_target": 23.98713657259941, |
| "summed_initial_full_target_nll": 44.80060669726497 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 412, |
| "initial_complete_functions": 23, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 22, |
| "actual_model_predictions": 50, |
| "actual_feedback_predictions": 18, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 74, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 23, |
| 0, |
| 5, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 23, |
| 23, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 412, |
| 443, |
| 481, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 579250, |
| "summed_initial_probability_mass_outside_full_target": 17.823746651411057, |
| "summed_initial_full_target_nll": 44.068231928590706 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 358, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 26, |
| "actual_model_predictions": 54, |
| "actual_feedback_predictions": 22, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 70, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 14, |
| 0, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 28, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 358, |
| 478, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 634499, |
| "summed_initial_probability_mass_outside_full_target": 17.94053691625595, |
| "summed_initial_full_target_nll": 44.5215218634449 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 145, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 145, |
| 283, |
| 394, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 834547, |
| "summed_initial_probability_mass_outside_full_target": 23.999182522296906, |
| "summed_initial_full_target_nll": 44.49356924337915 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 145, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 145, |
| 283, |
| 394, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 903583, |
| "summed_initial_probability_mass_outside_full_target": 23.991287380456924, |
| "summed_initial_full_target_nll": 44.45797759965371 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 158, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 158, |
| 289, |
| 400, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 838456, |
| "summed_initial_probability_mass_outside_full_target": 23.999674290418625, |
| "summed_initial_full_target_nll": 44.46622133538837 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 158, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 158, |
| 289, |
| 400, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 876827, |
| "summed_initial_probability_mass_outside_full_target": 23.992999494075775, |
| "summed_initial_full_target_nll": 44.44627728406336 |
| } |
| } |
| }, |
| "joint-qat_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2423, |
| "initial_complete_functions": 124, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 479, |
| "actual_model_predictions": 792, |
| "actual_feedback_predictions": 408, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 673, |
| "bilingual_initial_mask_disagreements": 114, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 124, |
| 112, |
| 77, |
| 71 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 124, |
| 236, |
| 313, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2423, |
| 3962, |
| 5178, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 9336370, |
| "summed_initial_probability_mass_outside_full_target": 264.97162987291813, |
| "summed_initial_full_target_nll": 525.6855548755702 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 283, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 56, |
| "actual_feedback_predictions": 24, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 0, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 283, |
| 399, |
| 399, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 660877, |
| "summed_initial_probability_mass_outside_full_target": 19.348411053419113, |
| "summed_initial_full_target_nll": 42.0239666770398 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 157, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 157, |
| 399, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 760127, |
| "summed_initial_probability_mass_outside_full_target": 19.84671440720558, |
| "summed_initial_full_target_nll": 43.96859888444768 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 197, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 38, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 58, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 6, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 197, |
| 328, |
| 421, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 735461, |
| "summed_initial_probability_mass_outside_full_target": 22.927854120731354, |
| "summed_initial_full_target_nll": 40.61118979666142 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 136, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 33, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 63, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 7, |
| 1 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 31, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 136, |
| 391, |
| 496, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 736664, |
| "summed_initial_probability_mass_outside_full_target": 23.95988169312477, |
| "summed_initial_full_target_nll": 45.189708459635185 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 148, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 49, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 47, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 7, |
| 9 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 23, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 148, |
| 278, |
| 381, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 828712, |
| "summed_initial_probability_mass_outside_full_target": 23.709769248962402, |
| "summed_initial_full_target_nll": 43.43819566358455 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 154, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 71, |
| "actual_feedback_predictions": 39, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 154, |
| 288, |
| 396, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 827622, |
| "summed_initial_probability_mass_outside_full_target": 23.902148693799973, |
| "summed_initial_full_target_nll": 45.18439960906941 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 353, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 30, |
| "actual_model_predictions": 58, |
| "actual_feedback_predictions": 26, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 66, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 10, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 353, |
| 401, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 712794, |
| "summed_initial_probability_mass_outside_full_target": 17.664475470781326, |
| "summed_initial_full_target_nll": 43.719917149763944 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 357, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 26, |
| "actual_model_predictions": 54, |
| "actual_feedback_predictions": 22, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 70, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 14, |
| 0, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 28, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 357, |
| 468, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 653075, |
| "summed_initial_probability_mass_outside_full_target": 17.800772562623024, |
| "summed_initial_full_target_nll": 44.59855760982592 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 138, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 138, |
| 258, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 851080, |
| "summed_initial_probability_mass_outside_full_target": 23.97306102514267, |
| "summed_initial_full_target_nll": 44.349513222215805 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 138, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 70, |
| "actual_feedback_predictions": 38, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 10, |
| 4, |
| 10 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 18, |
| 22, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 138, |
| 290, |
| 373, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 831960, |
| "summed_initial_probability_mass_outside_full_target": 23.9512600004673, |
| "summed_initial_full_target_nll": 44.19247254222447 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 145, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 50, |
| "actual_model_predictions": 75, |
| "actual_feedback_predictions": 43, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 46, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 5, |
| 12, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 13, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 145, |
| 239, |
| 415, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 875410, |
| "summed_initial_probability_mass_outside_full_target": 23.991551250219345, |
| "summed_initial_full_target_nll": 44.43531872986863 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 217, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 0, |
| 14, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 12, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 217, |
| 223, |
| 428, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 862588, |
| "summed_initial_probability_mass_outside_full_target": 23.89573034644127, |
| "summed_initial_full_target_nll": 43.97371653123351 |
| } |
| } |
| }, |
| "offline": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2318, |
| "initial_complete_functions": 116, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 516, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 636, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 116, |
| 100, |
| 88, |
| 80 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 116, |
| 216, |
| 304, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2318, |
| 3808, |
| 5038, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 268, |
| "summed_initial_full_target_nll": 7957.734081387455 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 222, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 222, |
| 396, |
| 455, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 20, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 222, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 222, |
| 396, |
| 455, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 20, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 134, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 134, |
| 265, |
| 390, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 134, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 134, |
| 265, |
| 390, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 276, |
| 397, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 276, |
| 397, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 354, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 34, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 62, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 6, |
| 8, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 20, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 354, |
| 416, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 18, |
| "summed_initial_full_target_nll": 663.1445067822847 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 354, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 34, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 62, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 6, |
| 8, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 20, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 354, |
| 416, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 18, |
| "summed_initial_full_target_nll": 663.1445067822847 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 141, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 141, |
| 273, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 141, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 141, |
| 273, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 153, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 153, |
| 278, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 153, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 153, |
| 278, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| } |
| } |
| } |
| }, |
| "dataset_sha256": "2d1c9888a648d590e77253666b6e5ec848d375e47a3c99daf29dbde8bcbc7383", |
| "default_model_promoted": false, |
| "development": { |
| "independent-fp32": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2596, |
| "initial_complete_functions": 124, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 448, |
| "actual_model_predictions": 1447, |
| "actual_feedback_predictions": 679, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 704, |
| "bilingual_initial_mask_disagreements": 182, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 124, |
| 130, |
| 72, |
| 58 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 124, |
| 254, |
| 326, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2596, |
| 4288, |
| 5356, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 13756082, |
| "summed_initial_probability_mass_outside_full_target": 270.2369040239579, |
| "summed_initial_full_target_nll": 529.3969786973563 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 282, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 4, |
| 4, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 20, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 282, |
| 335, |
| 396, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1089122, |
| "summed_initial_probability_mass_outside_full_target": 19.454412209402626, |
| "summed_initial_full_target_nll": 43.344522632874806 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 282, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 16, |
| "actual_model_predictions": 96, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 80, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 16, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 282, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 864500, |
| "summed_initial_probability_mass_outside_full_target": 20.139105324603825, |
| "summed_initial_full_target_nll": 45.59576241078689 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 150, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 127, |
| "actual_feedback_predictions": 63, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 150, |
| 284, |
| 395, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1173502, |
| "summed_initial_probability_mass_outside_full_target": 23.584299142719313, |
| "summed_initial_full_target_nll": 43.05492018665735 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 216, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 27, |
| "actual_model_predictions": 114, |
| "actual_feedback_predictions": 50, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 69, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 13, |
| 7, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 25, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 216, |
| 406, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1082830, |
| "summed_initial_probability_mass_outside_full_target": 24.03600345658598, |
| "summed_initial_full_target_nll": 44.86495176361099 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 152, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 131, |
| "actual_feedback_predictions": 67, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 8, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 152, |
| 295, |
| 413, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1207625, |
| "summed_initial_probability_mass_outside_full_target": 23.52249151649616, |
| "summed_initial_full_target_nll": 42.79543009501006 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 166, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 33, |
| "actual_model_predictions": 125, |
| "actual_feedback_predictions": 61, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 63, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 15, |
| 9, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 23, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 166, |
| 378, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1215378, |
| "summed_initial_probability_mass_outside_full_target": 24.195120364891046, |
| "summed_initial_full_target_nll": 45.297945959234454 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 349, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 112, |
| "actual_feedback_predictions": 48, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 349, |
| 398, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1119580, |
| "summed_initial_probability_mass_outside_full_target": 19.303478971726875, |
| "summed_initial_full_target_nll": 42.89452533331658 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 436, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 22, |
| "actual_model_predictions": 108, |
| "actual_feedback_predictions": 44, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 74, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 10, |
| 6, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 26, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 436, |
| 462, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 974000, |
| "summed_initial_probability_mass_outside_full_target": 20.172023861815838, |
| "summed_initial_full_target_nll": 44.67739308860217 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 84, |
| "initial_complete_functions": 4, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 51, |
| "actual_model_predictions": 135, |
| "actual_feedback_predictions": 71, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 45, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 4, |
| 13, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 4, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 84, |
| 305, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1296039, |
| "summed_initial_probability_mass_outside_full_target": 23.991596790878955, |
| "summed_initial_full_target_nll": 44.35048585661826 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 211, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 43, |
| "actual_model_predictions": 119, |
| "actual_feedback_predictions": 55, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 53, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 5, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 211, |
| 288, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1131835, |
| "summed_initial_probability_mass_outside_full_target": 23.921435067123383, |
| "summed_initial_full_target_nll": 44.06931229154658 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 108, |
| "initial_complete_functions": 4, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 50, |
| "actual_model_predictions": 135, |
| "actual_feedback_predictions": 71, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 46, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 4, |
| 13, |
| 8, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 4, |
| 17, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 108, |
| 291, |
| 417, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1311341, |
| "summed_initial_probability_mass_outside_full_target": 24.00850048699855, |
| "summed_initial_full_target_nll": 44.404286398846736 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 160, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 45, |
| "actual_model_predictions": 125, |
| "actual_feedback_predictions": 61, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 51, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 11, |
| 5, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 19, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 160, |
| 334, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1290330, |
| "summed_initial_probability_mass_outside_full_target": 23.908436830714706, |
| "summed_initial_full_target_nll": 44.04744268025131 |
| } |
| } |
| }, |
| "independent-ptq_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2307, |
| "initial_complete_functions": 108, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 479, |
| "actual_model_predictions": 1470, |
| "actual_feedback_predictions": 702, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 673, |
| "bilingual_initial_mask_disagreements": 136, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 108, |
| 131, |
| 87, |
| 58 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 108, |
| 239, |
| 326, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2307, |
| 4105, |
| 5380, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 15919801, |
| "summed_initial_probability_mass_outside_full_target": 270.8211417307867, |
| "summed_initial_full_target_nll": 531.008970892193 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 282, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 21, |
| "actual_model_predictions": 101, |
| "actual_feedback_predictions": 37, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 75, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 11, |
| 5, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 27, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 282, |
| 438, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1020247, |
| "summed_initial_probability_mass_outside_full_target": 19.58995282464997, |
| "summed_initial_full_target_nll": 43.94805563644665 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 156, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 31, |
| "actual_model_predictions": 119, |
| "actual_feedback_predictions": 55, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 65, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 17, |
| 7, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 25, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 156, |
| 415, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1281747, |
| "summed_initial_probability_mass_outside_full_target": 20.12548944983616, |
| "summed_initial_full_target_nll": 44.91999736054559 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 150, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 127, |
| "actual_feedback_predictions": 63, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 150, |
| 281, |
| 390, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1391378, |
| "summed_initial_probability_mass_outside_full_target": 23.713772109858684, |
| "summed_initial_full_target_nll": 43.64070089090548 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 37, |
| "actual_model_predictions": 125, |
| "actual_feedback_predictions": 61, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 59, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 11, |
| 13, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 19, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 314, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1313379, |
| "summed_initial_probability_mass_outside_full_target": 24.03457078098702, |
| "summed_initial_full_target_nll": 44.59113540272631 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 89, |
| "initial_complete_functions": 4, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 135, |
| "actual_feedback_predictions": 71, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 4, |
| 13, |
| 11, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 4, |
| 17, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 89, |
| 292, |
| 458, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1433665, |
| "summed_initial_probability_mass_outside_full_target": 23.678709436267663, |
| "summed_initial_full_target_nll": 43.464071335228425 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 215, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 40, |
| "actual_model_predictions": 118, |
| "actual_feedback_predictions": 54, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 56, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 6, |
| 8, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 18, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 215, |
| 308, |
| 427, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1248920, |
| "summed_initial_probability_mass_outside_full_target": 24.140005747662965, |
| "summed_initial_full_target_nll": 45.02279083272974 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 349, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 31, |
| "actual_model_predictions": 111, |
| "actual_feedback_predictions": 47, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 65, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 13, |
| 3, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 25, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 349, |
| 448, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1179077, |
| "summed_initial_probability_mass_outside_full_target": 19.5558538160392, |
| "summed_initial_full_target_nll": 43.60809592312091 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 349, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 29, |
| "actual_model_predictions": 109, |
| "actual_feedback_predictions": 45, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 67, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 15, |
| 1, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 27, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 349, |
| 460, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1185290, |
| "summed_initial_probability_mass_outside_full_target": 20.059972514073458, |
| "summed_initial_full_target_nll": 44.40973699463653 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 84, |
| "initial_complete_functions": 4, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 52, |
| "actual_model_predictions": 136, |
| "actual_feedback_predictions": 72, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 44, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 4, |
| 12, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 4, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 84, |
| 289, |
| 403, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1543422, |
| "summed_initial_probability_mass_outside_full_target": 24.000360483152786, |
| "summed_initial_full_target_nll": 44.45233420054842 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 288, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1420295, |
| "summed_initial_probability_mass_outside_full_target": 23.967537565014595, |
| "summed_initial_full_target_nll": 44.29192324151203 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 95, |
| "initial_complete_functions": 4, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 52, |
| "actual_model_predictions": 140, |
| "actual_feedback_predictions": 76, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 44, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 4, |
| 13, |
| 6, |
| 9 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 4, |
| 17, |
| 23, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 95, |
| 294, |
| 391, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1565501, |
| "summed_initial_probability_mass_outside_full_target": 23.996031074746778, |
| "summed_initial_full_target_nll": 44.373301873671785 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 228, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 44, |
| "actual_model_predictions": 121, |
| "actual_feedback_predictions": 57, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 52, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 3, |
| 10, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 15, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 228, |
| 278, |
| 418, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1336880, |
| "summed_initial_probability_mass_outside_full_target": 23.958885928497683, |
| "summed_initial_full_target_nll": 44.286827200121316 |
| } |
| } |
| }, |
| "independent-qat_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2509, |
| "initial_complete_functions": 120, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 455, |
| "actual_model_predictions": 1453, |
| "actual_feedback_predictions": 685, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 697, |
| "bilingual_initial_mask_disagreements": 182, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 120, |
| 131, |
| 75, |
| 58 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 120, |
| 251, |
| 326, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2509, |
| 4253, |
| 5367, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 15927855, |
| "summed_initial_probability_mass_outside_full_target": 270.4540155270739, |
| "summed_initial_full_target_nll": 529.3311030850092 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 282, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 33, |
| "actual_model_predictions": 110, |
| "actual_feedback_predictions": 46, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 63, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 7, |
| 1, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 23, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 282, |
| 381, |
| 396, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1141291, |
| "summed_initial_probability_mass_outside_full_target": 19.5702818550002, |
| "summed_initial_full_target_nll": 43.61249070486638 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 282, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 16, |
| "actual_model_predictions": 96, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 80, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 16, |
| 0, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 32, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 282, |
| 512, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 978378, |
| "summed_initial_probability_mass_outside_full_target": 20.098246110239913, |
| "summed_initial_full_target_nll": 45.20473649221639 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 150, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 127, |
| "actual_feedback_predictions": 63, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 10, |
| 6, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 18, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 150, |
| 299, |
| 392, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1393834, |
| "summed_initial_probability_mass_outside_full_target": 23.64181559662097, |
| "summed_initial_full_target_nll": 43.37011118107379 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 216, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 27, |
| "actual_model_predictions": 114, |
| "actual_feedback_predictions": 50, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 69, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 13, |
| 7, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 25, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 216, |
| 406, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1331119, |
| "summed_initial_probability_mass_outside_full_target": 23.973971136092583, |
| "summed_initial_full_target_nll": 44.46921377369318 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 152, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 130, |
| "actual_feedback_predictions": 66, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 152, |
| 295, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1407759, |
| "summed_initial_probability_mass_outside_full_target": 23.646611086088388, |
| "summed_initial_full_target_nll": 43.11466373324452 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 166, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 124, |
| "actual_feedback_predictions": 60, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 166, |
| 394, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1330336, |
| "summed_initial_probability_mass_outside_full_target": 24.14050346223522, |
| "summed_initial_full_target_nll": 45.10495157595851 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 262, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 43, |
| "actual_model_predictions": 128, |
| "actual_feedback_predictions": 64, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 53, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 12, |
| 5, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 20, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 262, |
| 364, |
| 454, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1441209, |
| "summed_initial_probability_mass_outside_full_target": 19.517105786555494, |
| "summed_initial_full_target_nll": 43.2892556697893 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 436, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 20, |
| "actual_model_predictions": 104, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 76, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 12, |
| 4, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 28, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 436, |
| 478, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1095418, |
| "summed_initial_probability_mass_outside_full_target": 20.015601767231146, |
| "summed_initial_full_target_nll": 44.16782908120292 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 84, |
| "initial_complete_functions": 4, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 52, |
| "actual_model_predictions": 137, |
| "actual_feedback_predictions": 73, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 44, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 4, |
| 11, |
| 10, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 4, |
| 15, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 84, |
| 269, |
| 417, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1527051, |
| "summed_initial_probability_mass_outside_full_target": 23.994678123135266, |
| "summed_initial_full_target_nll": 44.37544120572716 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 211, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 42, |
| "actual_model_predictions": 120, |
| "actual_feedback_predictions": 56, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 54, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 4, |
| 10, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 16, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 211, |
| 272, |
| 425, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1351379, |
| "summed_initial_probability_mass_outside_full_target": 23.941040831911284, |
| "summed_initial_full_target_nll": 44.154172901818406 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 108, |
| "initial_complete_functions": 4, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 50, |
| "actual_model_predictions": 136, |
| "actual_feedback_predictions": 72, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 46, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 4, |
| 12, |
| 10, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 4, |
| 16, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 108, |
| 277, |
| 432, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1529791, |
| "summed_initial_probability_mass_outside_full_target": 24.004105441701263, |
| "summed_initial_full_target_nll": 44.39690641397221 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 160, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 47, |
| "actual_model_predictions": 127, |
| "actual_feedback_predictions": 63, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 49, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 9, |
| 7, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 17, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 160, |
| 306, |
| 405, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 1400290, |
| "summed_initial_probability_mass_outside_full_target": 23.910054330261914, |
| "summed_initial_full_target_nll": 44.07133035144721 |
| } |
| } |
| }, |
| "joint-fp32": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2639, |
| "initial_complete_functions": 128, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 455, |
| "actual_model_predictions": 777, |
| "actual_feedback_predictions": 393, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 697, |
| "bilingual_initial_mask_disagreements": 180, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 128, |
| 119, |
| 75, |
| 62 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 128, |
| 247, |
| 322, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2639, |
| 4171, |
| 5291, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 9500791, |
| "summed_initial_probability_mass_outside_full_target": 271.0070815384388, |
| "summed_initial_full_target_nll": 530.778929149537 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 395, |
| "initial_complete_functions": 24, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 14, |
| "actual_model_predictions": 46, |
| "actual_feedback_predictions": 14, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 82, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 24, |
| 2, |
| 6, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 24, |
| 26, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 395, |
| 425, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 529042, |
| "summed_initial_probability_mass_outside_full_target": 19.70918470621109, |
| "summed_initial_full_target_nll": 43.584187896788585 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 282, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 24, |
| "actual_model_predictions": 56, |
| "actual_feedback_predictions": 24, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 72, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 282, |
| 399, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 687464, |
| "summed_initial_probability_mass_outside_full_target": 19.967933505773544, |
| "summed_initial_full_target_nll": 44.87374996803259 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 146, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 146, |
| 262, |
| 385, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 851002, |
| "summed_initial_probability_mass_outside_full_target": 23.63910436630249, |
| "summed_initial_full_target_nll": 43.25037554821077 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 140, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 37, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 59, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 3, |
| 5 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 27, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 140, |
| 388, |
| 438, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 768960, |
| "summed_initial_probability_mass_outside_full_target": 24.081539511680603, |
| "summed_initial_full_target_nll": 44.843781387203016 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 121, |
| "initial_complete_functions": 6, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 54, |
| "actual_model_predictions": 76, |
| "actual_feedback_predictions": 44, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 42, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 6, |
| 8, |
| 8, |
| 10 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 6, |
| 14, |
| 22, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 121, |
| 248, |
| 363, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 903829, |
| "summed_initial_probability_mass_outside_full_target": 23.72674262523651, |
| "summed_initial_full_target_nll": 43.436827578169385 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 166, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 166, |
| 394, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 748539, |
| "summed_initial_probability_mass_outside_full_target": 24.1494023501873, |
| "summed_initial_full_target_nll": 45.07644758544018 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 349, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 60, |
| "actual_feedback_predictions": 28, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 349, |
| 398, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 745418, |
| "summed_initial_probability_mass_outside_full_target": 19.61037689447403, |
| "summed_initial_full_target_nll": 43.479983873128 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 436, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 20, |
| "actual_model_predictions": 52, |
| "actual_feedback_predictions": 20, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 76, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 12, |
| 4, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 28, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 436, |
| 478, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 629457, |
| "summed_initial_probability_mass_outside_full_target": 20.16948327422142, |
| "summed_initial_full_target_nll": 44.838219134947124 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 126, |
| "initial_complete_functions": 6, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 50, |
| "actual_model_predictions": 74, |
| "actual_feedback_predictions": 42, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 46, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 6, |
| 10, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 6, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 126, |
| 281, |
| 406, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 901786, |
| "summed_initial_probability_mass_outside_full_target": 23.999244272708893, |
| "summed_initial_full_target_nll": 44.370327581345634 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 189, |
| "initial_complete_functions": 10, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 42, |
| "actual_model_predictions": 67, |
| "actual_feedback_predictions": 35, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 54, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 10, |
| 9, |
| 6, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 10, |
| 19, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 189, |
| 331, |
| 417, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 958752, |
| "summed_initial_probability_mass_outside_full_target": 23.98909765481949, |
| "summed_initial_full_target_nll": 44.366243987301765 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 129, |
| "initial_complete_functions": 6, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 54, |
| "actual_model_predictions": 74, |
| "actual_feedback_predictions": 42, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 42, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 6, |
| 10, |
| 4, |
| 12 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 6, |
| 16, |
| 20, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 129, |
| 291, |
| 352, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 921792, |
| "summed_initial_probability_mass_outside_full_target": 23.994300425052643, |
| "summed_initial_full_target_nll": 44.351964492171916 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 160, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 160, |
| 276, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 854750, |
| "summed_initial_probability_mass_outside_full_target": 23.970671951770782, |
| "summed_initial_full_target_nll": 44.3068201167975 |
| } |
| } |
| }, |
| "joint-ptq_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2635, |
| "initial_complete_functions": 128, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 485, |
| "actual_model_predictions": 808, |
| "actual_feedback_predictions": 424, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 667, |
| "bilingual_initial_mask_disagreements": 24, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 128, |
| 88, |
| 107, |
| 61 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 128, |
| 216, |
| 323, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2635, |
| 3927, |
| 5306, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 9913474, |
| "summed_initial_probability_mass_outside_full_target": 271.1784009039402, |
| "summed_initial_full_target_nll": 533.2008575664042 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 282, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 40, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 56, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 0, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 282, |
| 282, |
| 396, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 740500, |
| "summed_initial_probability_mass_outside_full_target": 19.624286621809006, |
| "summed_initial_full_target_nll": 44.11965010645702 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 282, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 28, |
| "actual_model_predictions": 60, |
| "actual_feedback_predictions": 28, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 68, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 4, |
| 12, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 20, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 282, |
| 342, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 719296, |
| "summed_initial_probability_mass_outside_full_target": 19.772160291671753, |
| "summed_initial_full_target_nll": 44.774355854440785 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 265, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 39, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 57, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 0, |
| 9, |
| 7 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 16, |
| 25, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 265, |
| 277, |
| 403, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 774707, |
| "summed_initial_probability_mass_outside_full_target": 23.848025798797607, |
| "summed_initial_full_target_nll": 43.93552281976686 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 140, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 68, |
| "actual_feedback_predictions": 36, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 12, |
| 12, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 20, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 140, |
| 339, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 828751, |
| "summed_initial_probability_mass_outside_full_target": 24.031789749860764, |
| "summed_initial_full_target_nll": 44.65230342003479 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 158, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 158, |
| 281, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 843124, |
| "summed_initial_probability_mass_outside_full_target": 23.897545754909515, |
| "summed_initial_full_target_nll": 44.09979402983785 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 158, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 35, |
| "actual_model_predictions": 67, |
| "actual_feedback_predictions": 35, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 61, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 13, |
| 11, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 21, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 158, |
| 349, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 835251, |
| "summed_initial_probability_mass_outside_full_target": 24.07825180888176, |
| "summed_initial_full_target_nll": 44.875306904286916 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 353, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 37, |
| "actual_model_predictions": 65, |
| "actual_feedback_predictions": 33, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 59, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 7, |
| 9, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 19, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 353, |
| 439, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 777416, |
| "summed_initial_probability_mass_outside_full_target": 19.853339344263077, |
| "summed_initial_full_target_nll": 44.14603806517079 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 353, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 28, |
| "actual_model_predictions": 60, |
| "actual_feedback_predictions": 28, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 68, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 353, |
| 444, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 757420, |
| "summed_initial_probability_mass_outside_full_target": 20.084148108959198, |
| "summed_initial_full_target_nll": 44.71109173647279 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 158, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 158, |
| 292, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 870631, |
| "summed_initial_probability_mass_outside_full_target": 24.00152823328972, |
| "summed_initial_full_target_nll": 44.495768449591935 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 158, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 158, |
| 292, |
| 401, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 907624, |
| "summed_initial_probability_mass_outside_full_target": 23.994916886091232, |
| "summed_initial_full_target_nll": 44.47724179397162 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 296, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 873294, |
| "summed_initial_probability_mass_outside_full_target": 24.00063782930374, |
| "summed_initial_full_target_nll": 44.46258006811767 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 164, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 50, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 46, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 6, |
| 10 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 22, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 164, |
| 294, |
| 377, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 985460, |
| "summed_initial_probability_mass_outside_full_target": 23.99177047610283, |
| "summed_initial_full_target_nll": 44.45120431825544 |
| } |
| } |
| }, |
| "joint-qat_ternary": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2394, |
| "initial_complete_functions": 114, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 469, |
| "actual_model_predictions": 792, |
| "actual_feedback_predictions": 408, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 683, |
| "bilingual_initial_mask_disagreements": 132, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 114, |
| 132, |
| 77, |
| 61 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 114, |
| 246, |
| 323, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2394, |
| 4132, |
| 5313, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 9659504, |
| "summed_initial_probability_mass_outside_full_target": 271.5664424300194, |
| "summed_initial_full_target_nll": 531.9454298586631 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 282, |
| "initial_complete_functions": 16, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 30, |
| "actual_model_predictions": 56, |
| "actual_feedback_predictions": 24, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 66, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 16, |
| 8, |
| 2, |
| 6 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 16, |
| 24, |
| 26, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 282, |
| 400, |
| 425, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 661082, |
| "summed_initial_probability_mass_outside_full_target": 19.756559431552887, |
| "summed_initial_full_target_nll": 43.62048001899949 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 156, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 156, |
| 400, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 811495, |
| "summed_initial_probability_mass_outside_full_target": 20.210161596536636, |
| "summed_initial_full_target_nll": 45.039872947373944 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 150, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 150, |
| 268, |
| 391, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 860958, |
| "summed_initial_probability_mass_outside_full_target": 23.672243654727936, |
| "summed_initial_full_target_nll": 43.603786318080616 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 35, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 61, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 5, |
| 3 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 29, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 391, |
| 465, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 781752, |
| "summed_initial_probability_mass_outside_full_target": 24.098874390125275, |
| "summed_initial_full_target_nll": 44.84797966183368 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 152, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 152, |
| 282, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 854839, |
| "summed_initial_probability_mass_outside_full_target": 23.76179540157318, |
| "summed_initial_full_target_nll": 43.50203847362654 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 162, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 64, |
| "actual_feedback_predictions": 32, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 16, |
| 8, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 24, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 162, |
| 396, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 782831, |
| "summed_initial_probability_mass_outside_full_target": 24.226962327957153, |
| "summed_initial_full_target_nll": 45.36419512755738 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 358, |
| "initial_complete_functions": 14, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 26, |
| "actual_model_predictions": 54, |
| "actual_feedback_predictions": 22, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 70, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 14, |
| 14, |
| 0, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 14, |
| 28, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 358, |
| 453, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 629581, |
| "summed_initial_probability_mass_outside_full_target": 19.677556931972504, |
| "summed_initial_full_target_nll": 43.65170114181307 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 353, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 24, |
| "actual_model_predictions": 56, |
| "actual_feedback_predictions": 24, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 72, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 16, |
| 4, |
| 0 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 28, |
| 32, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 353, |
| 453, |
| 512, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 703040, |
| "summed_initial_probability_mass_outside_full_target": 20.269226163625717, |
| "summed_initial_full_target_nll": 45.06549546193283 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 153, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 153, |
| 269, |
| 406, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 890833, |
| "summed_initial_probability_mass_outside_full_target": 23.9886714220047, |
| "summed_initial_full_target_nll": 44.38655418957031 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 153, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 46, |
| "actual_model_predictions": 70, |
| "actual_feedback_predictions": 38, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 50, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 10, |
| 6, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 18, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 153, |
| 302, |
| 406, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 864915, |
| "summed_initial_probability_mass_outside_full_target": 23.983598321676254, |
| "summed_initial_full_target_nll": 44.31255996050225 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 160, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 52, |
| "actual_model_predictions": 76, |
| "actual_feedback_predictions": 44, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 44, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 4, |
| 12, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 12, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 160, |
| 235, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 930254, |
| "summed_initial_probability_mass_outside_full_target": 23.9864062666893, |
| "summed_initial_full_target_nll": 44.38171781258787 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 160, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 72, |
| "actual_feedback_predictions": 40, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 160, |
| 283, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 887924, |
| "summed_initial_probability_mass_outside_full_target": 23.934386521577835, |
| "summed_initial_full_target_nll": 44.16904874478554 |
| } |
| } |
| }, |
| "offline": { |
| "total": { |
| "function_views": 384, |
| "ordered_finite_cases": 6144, |
| "initial_cases_passed": 2394, |
| "initial_complete_functions": 112, |
| "complete_functions_by_budget_four": 384, |
| "additional_candidate_attempts": 520, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 632, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 192, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 112, |
| 104, |
| 88, |
| 80 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 112, |
| 216, |
| 304, |
| 384 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 2394, |
| 3854, |
| 5052, |
| 6144 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 272, |
| "summed_initial_full_target_nll": 7957.734081387455 |
| }, |
| "family_language": { |
| "assignment_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 221, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 221, |
| 397, |
| 456, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 20, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "assignment_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 221, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 32, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 64, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 12, |
| 4, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 24, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 221, |
| 397, |
| 456, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 20, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "branch_reference/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 154, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 154, |
| 267, |
| 390, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "branch_reference/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 154, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 154, |
| 267, |
| 390, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "operand_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 281, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "operand_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 155, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 155, |
| 281, |
| 398, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "predicate_assignment/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 351, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 8, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 351, |
| 417, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 20, |
| "summed_initial_full_target_nll": 663.1445067822849 |
| }, |
| "predicate_assignment/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 351, |
| "initial_complete_functions": 12, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 36, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 60, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 12, |
| 8, |
| 8, |
| 4 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 12, |
| 20, |
| 28, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 351, |
| 417, |
| 478, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 20, |
| "summed_initial_full_target_nll": 663.1445067822849 |
| }, |
| "schedule_branch/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 156, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 156, |
| 280, |
| 400, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "schedule_branch/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 156, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 156, |
| 280, |
| 400, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "schedule_operand/en": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 160, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 160, |
| 285, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| }, |
| "schedule_operand/ko": { |
| "function_views": 32, |
| "ordered_finite_cases": 512, |
| "initial_cases_passed": 160, |
| "initial_complete_functions": 8, |
| "complete_functions_by_budget_four": 32, |
| "additional_candidate_attempts": 48, |
| "actual_model_predictions": 0, |
| "actual_feedback_predictions": 0, |
| "zero_call_feedback_context_declines": 0, |
| "type_rejections": 0, |
| "unattempted_masks_after_completion": 48, |
| "bilingual_initial_mask_disagreements": 0, |
| "bilingual_function_pairs": 0, |
| "minimum_complete_budget_histogram_1_to_4": [ |
| 8, |
| 8, |
| 8, |
| 8 |
| ], |
| "complete_function_curve_budgets_1_to_4": [ |
| 8, |
| 16, |
| 24, |
| 32 |
| ], |
| "best_passing_case_curve_budgets_1_to_4": [ |
| 160, |
| 285, |
| 404, |
| 512 |
| ], |
| "summed_observed_prediction_ns": 0, |
| "summed_initial_probability_mass_outside_full_target": 24, |
| "summed_initial_full_target_nll": 663.1445067822851 |
| } |
| } |
| } |
| }, |
| "go_python_parity_max_absolute_error": { |
| "independent": 8.940696716308594e-8, |
| "joint": 1.601874828338623e-7 |
| }, |
| "native_calls": 0, |
| "new_optimizer_updates": 0, |
| "parity_predictions": 192, |
| "runtime": { |
| "independent-fp32": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 8932.667, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 50912, |
| "matrix_scale_bytes": 0, |
| "workspace_bytes": 1248 |
| }, |
| "independent-ptq_ternary": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 9935.583, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 12896, |
| "matrix_scale_bytes": 8, |
| "workspace_bytes": 1248 |
| }, |
| "independent-qat_ternary": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 10035.084, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 12896, |
| "matrix_scale_bytes": 8, |
| "workspace_bytes": 1248 |
| }, |
| "joint-fp32": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 10694.792, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 49648, |
| "matrix_scale_bytes": 0, |
| "workspace_bytes": 2160 |
| }, |
| "joint-ptq_ternary": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 10649, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 12496, |
| "matrix_scale_bytes": 8, |
| "workspace_bytes": 2160 |
| }, |
| "joint-qat_ternary": { |
| "warm_benchmark_predictions": 1001, |
| "warm_ns_per_prediction": 10592.958, |
| "predict_into_heap_allocations_per_call": 0, |
| "allocation_probe_predictions": 1001, |
| "resident_tensor_bytes": 12496, |
| "matrix_scale_bytes": 8, |
| "workspace_bytes": 2160 |
| } |
| }, |
| "schema": "gooo/joint-composition-study/v1", |
| "scope": "Six compositions, four goals with bilingual/parameter variants; all 16 ordered independent cases retained. Model ranks legal compiler-owned fragments and observed feedback only. Calibration selector frozen before development sessions. Four-path completeness is finite enumerative evidence, not general language understanding. Warm inference allocations exclude loading/context/session and decoded ternary matrices remain int8.", |
| "selected_candidate": "independent-fp32", |
| "source_revision": "b775ba63b81f20ec585c33a8c39f4e43b8cf5986", |
| "status": "PASS" |
| } |
|
|
