| { | |
| "passed": true, | |
| "total_rows": 16428, | |
| "tasks": { | |
| "hellaswag": { | |
| "rows": 10042, | |
| "native_correct": 2809, | |
| "assisted_correct": 2805, | |
| "changed_system_predictions": 0, | |
| "helper_answers": 0, | |
| "repeated_source_id_rows": 433 | |
| }, | |
| "arc_easy": { | |
| "rows": 2376, | |
| "native_correct": 632, | |
| "assisted_correct": 1476, | |
| "changed_system_predictions": 0, | |
| "helper_answers": 0, | |
| "repeated_source_id_rows": 0 | |
| }, | |
| "arc_challenge": { | |
| "rows": 1172, | |
| "native_correct": 260, | |
| "assisted_correct": 371, | |
| "changed_system_predictions": 0, | |
| "helper_answers": 0, | |
| "repeated_source_id_rows": 0 | |
| }, | |
| "piqa": { | |
| "rows": 1838, | |
| "native_correct": 899, | |
| "assisted_correct": 1078, | |
| "changed_system_predictions": 0, | |
| "helper_answers": 0, | |
| "repeated_source_id_rows": 0 | |
| }, | |
| "arithmark3": { | |
| "rows": 1000, | |
| "native_correct": 378, | |
| "assisted_correct": 695, | |
| "changed_system_predictions": 0, | |
| "helper_answers": 0, | |
| "repeated_source_id_rows": 0 | |
| } | |
| }, | |
| "fresh_sql_parity_checks": 15, | |
| "maximum_native_likelihood_error": 0.0, | |
| "source_and_asset_hashes_verified": true, | |
| "all_predictions_rescored": true | |
| } | |