| { | |
| "check": { | |
| "snippet": 60, | |
| "gap": 100, | |
| "look_back": 6000, | |
| "min_count": 3 | |
| }, | |
| "runs": { | |
| "bench/base/math500": { | |
| "answers": 2000, | |
| "correct": 146, | |
| "groups": { | |
| "cut while thinking": { | |
| "answers": 1863, | |
| "correct": 13 | |
| }, | |
| "stopped while thinking": { | |
| "answers": 0, | |
| "correct": 0 | |
| }, | |
| "closed out of a loop": { | |
| "answers": 8, | |
| "correct": 4 | |
| }, | |
| "closed normally": { | |
| "answers": 129, | |
| "correct": 129 | |
| } | |
| } | |
| }, | |
| "bench/p50c/math500": { | |
| "answers": 2000, | |
| "correct": 1036, | |
| "groups": { | |
| "cut while thinking": { | |
| "answers": 416, | |
| "correct": 1 | |
| }, | |
| "stopped while thinking": { | |
| "answers": 1, | |
| "correct": 0 | |
| }, | |
| "closed out of a loop": { | |
| "answers": 581, | |
| "correct": 191 | |
| }, | |
| "closed normally": { | |
| "answers": 1002, | |
| "correct": 844 | |
| } | |
| } | |
| }, | |
| "bench/base/gsm8k": { | |
| "answers": 2638, | |
| "correct": 374, | |
| "groups": { | |
| "cut while thinking": { | |
| "answers": 2312, | |
| "correct": 57 | |
| }, | |
| "stopped while thinking": { | |
| "answers": 0, | |
| "correct": 0 | |
| }, | |
| "closed out of a loop": { | |
| "answers": 38, | |
| "correct": 36 | |
| }, | |
| "closed normally": { | |
| "answers": 288, | |
| "correct": 281 | |
| } | |
| } | |
| }, | |
| "bench/p50c/gsm8k": { | |
| "answers": 2638, | |
| "correct": 1765, | |
| "groups": { | |
| "cut while thinking": { | |
| "answers": 116, | |
| "correct": 0 | |
| }, | |
| "stopped while thinking": { | |
| "answers": 1, | |
| "correct": 0 | |
| }, | |
| "closed out of a loop": { | |
| "answers": 558, | |
| "correct": 190 | |
| }, | |
| "closed normally": { | |
| "answers": 1963, | |
| "correct": 1575 | |
| } | |
| } | |
| }, | |
| "bench/base/ifeval": { | |
| "answers": 1082, | |
| "correct": 229, | |
| "groups": { | |
| "cut while thinking": { | |
| "answers": 788, | |
| "correct": 0 | |
| }, | |
| "stopped while thinking": { | |
| "answers": 0, | |
| "correct": 0 | |
| }, | |
| "closed out of a loop": { | |
| "answers": 18, | |
| "correct": 15 | |
| }, | |
| "closed normally": { | |
| "answers": 276, | |
| "correct": 214 | |
| } | |
| } | |
| }, | |
| "bench/p50c/ifeval": { | |
| "answers": 1082, | |
| "correct": 686, | |
| "groups": { | |
| "cut while thinking": { | |
| "answers": 10, | |
| "correct": 0 | |
| }, | |
| "stopped while thinking": { | |
| "answers": 2, | |
| "correct": 0 | |
| }, | |
| "closed out of a loop": { | |
| "answers": 58, | |
| "correct": 23 | |
| }, | |
| "closed normally": { | |
| "answers": 1012, | |
| "correct": 663 | |
| } | |
| } | |
| } | |
| } | |
| } |