diff --git "a/evaluation-items.json" "b/evaluation-items.json" new file mode 100644--- /dev/null +++ "b/evaluation-items.json" @@ -0,0 +1,100144 @@ +{ + "Qwen3.8-27B-w4a16-vision-run1": { + "started": "2026-08-30 06:17:05", + "finished": "2026-08-30 07:57:21", + "seed": 20260816, + "thinking": true, + "tasks": { + "mmlu": { + "task": "mmlu", + "n": 150, + "hits": 130, + "accuracy_pct": 86.67, + "accuracy_ex_trunc_pct": 90.28, + "truncated": 6, + "completion_tokens_p50": 265, + "completion_tokens_p90": 6062, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 747.3, + "gen_tokens": 416314, + "tok_per_s": 557.1, + "details": [ + { + "id": "mmlu-12325", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-154", + "ok": true, + "error": null, + "completion_tokens": 111, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12844", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-1153", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11089", + "ok": true, + "error": null, + "completion_tokens": 443, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12782", + "ok": true, + "error": null, + "completion_tokens": 867, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5422", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12363", + "ok": true, + "error": null, + "completion_tokens": 8756, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12556", + "ok": true, + "error": null, + "completion_tokens": 338, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13163", + "ok": true, + "error": null, + "completion_tokens": 2004, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-270", + "ok": true, + "error": null, + "completion_tokens": 300, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6943", + "ok": true, + "error": null, + "completion_tokens": 659, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8330", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9276", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-7584", + "ok": true, + "error": null, + "completion_tokens": 51, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12819", + "ok": true, + "error": null, + "completion_tokens": 476, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6163", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4099", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10389", + "ok": false, + "error": null, + "completion_tokens": 6062, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1539", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12013", + "ok": true, + "error": null, + "completion_tokens": 18388, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-7814", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1106", + "ok": true, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1616", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-555", + "ok": true, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3698", + "ok": true, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-618", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9842", + "ok": true, + "error": null, + "completion_tokens": 300, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4229", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1022", + "ok": true, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6421", + "ok": true, + "error": null, + "completion_tokens": 724, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1997", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8941", + "ok": false, + "error": null, + "completion_tokens": 620, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-565", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11364", + "ok": true, + "error": null, + "completion_tokens": 5415, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5959", + "ok": true, + "error": null, + "completion_tokens": 391, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13066", + "ok": false, + "error": null, + "completion_tokens": 3965, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3122", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3722", + "ok": true, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13839", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12570", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9959", + "ok": true, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10791", + "ok": true, + "error": null, + "completion_tokens": 15151, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12268", + "ok": true, + "error": null, + "completion_tokens": 772, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9051", + "ok": true, + "error": null, + "completion_tokens": 652, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12887", + "ok": true, + "error": null, + "completion_tokens": 789, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2667", + "ok": true, + "error": null, + "completion_tokens": 5927, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3906", + "ok": true, + "error": null, + "completion_tokens": 413, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10304", + "ok": true, + "error": null, + "completion_tokens": 51, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12927", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6337", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3907", + "ok": true, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12888", + "ok": false, + "error": null, + "completion_tokens": 377, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10629", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5801", + "ok": true, + "error": null, + "completion_tokens": 80, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2936", + "ok": true, + "error": null, + "completion_tokens": 254, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-7389", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9532", + "ok": false, + "error": null, + "completion_tokens": 2492, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8705", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11776", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-6836", + "ok": true, + "error": null, + "completion_tokens": 3171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-7856", + "ok": true, + "error": null, + "completion_tokens": 1187, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9697", + "ok": true, + "error": null, + "completion_tokens": 346, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10825", + "ok": true, + "error": null, + "completion_tokens": 7241, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11178", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-11420", + "ok": true, + "error": null, + "completion_tokens": 3772, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4658", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5403", + "ok": false, + "error": null, + "completion_tokens": 1864, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6160", + "ok": true, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6872", + "ok": false, + "error": null, + "completion_tokens": 15877, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9738", + "ok": true, + "error": null, + "completion_tokens": 294, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3076", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13496", + "ok": true, + "error": null, + "completion_tokens": 318, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11289", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-222", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12180", + "ok": true, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1143", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13086", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-7038", + "ok": true, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13587", + "ok": false, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12349", + "ok": true, + "error": null, + "completion_tokens": 320, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11747", + "ok": true, + "error": null, + "completion_tokens": 269, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9848", + "ok": false, + "error": null, + "completion_tokens": 11430, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3438", + "ok": true, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-89", + "ok": true, + "error": null, + "completion_tokens": 265, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4253", + "ok": true, + "error": null, + "completion_tokens": 116, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5460", + "ok": true, + "error": null, + "completion_tokens": 1308, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2494", + "ok": false, + "error": null, + "completion_tokens": 15659, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11149", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-718", + "ok": true, + "error": null, + "completion_tokens": 1159, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5330", + "ok": false, + "error": null, + "completion_tokens": 3801, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10716", + "ok": true, + "error": null, + "completion_tokens": 870, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5636", + "ok": true, + "error": null, + "completion_tokens": 384, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11832", + "ok": true, + "error": null, + "completion_tokens": 1167, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-95", + "ok": true, + "error": null, + "completion_tokens": 515, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13960", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2072", + "ok": true, + "error": null, + "completion_tokens": 76, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12030", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-2931", + "ok": true, + "error": null, + "completion_tokens": 200, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2441", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13666", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6575", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11625", + "ok": true, + "error": null, + "completion_tokens": 926, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13283", + "ok": false, + "error": null, + "completion_tokens": 11641, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4262", + "ok": true, + "error": null, + "completion_tokens": 325, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12931", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12150", + "ok": true, + "error": null, + "completion_tokens": 6002, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3678", + "ok": true, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4593", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1563", + "ok": true, + "error": null, + "completion_tokens": 2862, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13396", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8049", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8396", + "ok": true, + "error": null, + "completion_tokens": 606, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8493", + "ok": true, + "error": null, + "completion_tokens": 662, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13514", + "ok": true, + "error": null, + "completion_tokens": 300, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3000", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1635", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1391", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-627", + "ok": false, + "error": null, + "completion_tokens": 17817, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1573", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6731", + "ok": true, + "error": null, + "completion_tokens": 191, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11665", + "ok": true, + "error": null, + "completion_tokens": 3955, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2914", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6875", + "ok": true, + "error": null, + "completion_tokens": 3859, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2184", + "ok": true, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12801", + "ok": true, + "error": null, + "completion_tokens": 350, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6717", + "ok": true, + "error": null, + "completion_tokens": 542, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11", + "ok": true, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12464", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2787", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11549", + "ok": true, + "error": null, + "completion_tokens": 487, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-842", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13868", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9647", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9502", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12441", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4392", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5036", + "ok": true, + "error": null, + "completion_tokens": 740, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12606", + "ok": true, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1400", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13979", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13634", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11226", + "ok": true, + "error": null, + "completion_tokens": 4747, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-625", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-257", + "ok": true, + "error": null, + "completion_tokens": 555, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12853", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-451", + "ok": true, + "error": null, + "completion_tokens": 376, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2074", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9117", + "ok": false, + "error": null, + "completion_tokens": 2752, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10748", + "ok": true, + "error": null, + "completion_tokens": 364, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "cmmlu": { + "task": "cmmlu", + "n": 150, + "hits": 135, + "accuracy_pct": 90.0, + "accuracy_ex_trunc_pct": 91.22, + "truncated": 2, + "completion_tokens_p50": 338, + "completion_tokens_p90": 2729, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 573.1, + "gen_tokens": 213946, + "tok_per_s": 373.3, + "details": [ + { + "id": "cmmlu-chinese_teacher_qualification-63", + "ok": true, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-54", + "ok": true, + "error": null, + "completion_tokens": 5084, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_science-91", + "ok": true, + "error": null, + "completion_tokens": 543, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-260", + "ok": false, + "error": null, + "completion_tokens": 615, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-32", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-29", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-200", + "ok": true, + "error": null, + "completion_tokens": 5133, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-98", + "ok": true, + "error": null, + "completion_tokens": 366, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-44", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-118", + "ok": true, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-116", + "ok": true, + "error": null, + "completion_tokens": 3419, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-43", + "ok": true, + "error": null, + "completion_tokens": 282, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-145", + "ok": true, + "error": null, + "completion_tokens": 276, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-247", + "ok": true, + "error": null, + "completion_tokens": 388, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sociology-224", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-26", + "ok": true, + "error": null, + "completion_tokens": 882, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-105", + "ok": true, + "error": null, + "completion_tokens": 1547, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-logical-15", + "ok": true, + "error": null, + "completion_tokens": 264, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-62", + "ok": true, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-nutrition-74", + "ok": true, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-181", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-127", + "ok": true, + "error": null, + "completion_tokens": 727, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_geography-107", + "ok": true, + "error": null, + "completion_tokens": 841, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-agronomy-18", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-82", + "ok": true, + "error": null, + "completion_tokens": 156, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_food_culture-38", + "ok": true, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-198", + "ok": true, + "error": null, + "completion_tokens": 385, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-96", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-73", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "cmmlu-genetics-20", + "ok": true, + "error": null, + "completion_tokens": 273, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-logical-95", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-genetics-88", + "ok": true, + "error": null, + "completion_tokens": 763, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-192", + "ok": true, + "error": null, + "completion_tokens": 639, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-66", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-100", + "ok": true, + "error": null, + "completion_tokens": 320, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-136", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-management-16", + "ok": false, + "error": null, + "completion_tokens": 394, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-logical-90", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sports_science-41", + "ok": true, + "error": null, + "completion_tokens": 3287, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-345", + "ok": true, + "error": null, + "completion_tokens": 457, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sociology-103", + "ok": false, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-agronomy-93", + "ok": true, + "error": null, + "completion_tokens": 441, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-88", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-management-206", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-105", + "ok": true, + "error": null, + "completion_tokens": 802, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-economics-90", + "ok": true, + "error": null, + "completion_tokens": 86, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-190", + "ok": false, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-international_law-39", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-155", + "ok": true, + "error": null, + "completion_tokens": 1160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-24", + "ok": true, + "error": null, + "completion_tokens": 262, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-6", + "ok": true, + "error": null, + "completion_tokens": 422, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-31", + "ok": true, + "error": null, + "completion_tokens": 574, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-243", + "ok": true, + "error": null, + "completion_tokens": 665, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-63", + "ok": true, + "error": null, + "completion_tokens": 3117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sports_science-135", + "ok": true, + "error": null, + "completion_tokens": 601, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_science-89", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-91", + "ok": false, + "error": null, + "completion_tokens": 2729, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-120", + "ok": true, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-genetics-62", + "ok": true, + "error": null, + "completion_tokens": 1105, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-management-106", + "ok": true, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-food_science-106", + "ok": true, + "error": null, + "completion_tokens": 14995, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-24", + "ok": true, + "error": null, + "completion_tokens": 1793, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-157", + "ok": true, + "error": null, + "completion_tokens": 584, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marketing-41", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-115", + "ok": true, + "error": null, + "completion_tokens": 6875, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-journalism-29", + "ok": false, + "error": null, + "completion_tokens": 4632, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-67", + "ok": true, + "error": null, + "completion_tokens": 371, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-161", + "ok": true, + "error": null, + "completion_tokens": 733, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-virology-120", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-education-117", + "ok": true, + "error": null, + "completion_tokens": 84, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-169", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-129", + "ok": true, + "error": null, + "completion_tokens": 415, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-19", + "ok": true, + "error": null, + "completion_tokens": 1766, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-8", + "ok": true, + "error": null, + "completion_tokens": 202, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marketing-72", + "ok": true, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-public_relations-139", + "ok": false, + "error": null, + "completion_tokens": 862, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_security-22", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-7", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-108", + "ok": true, + "error": null, + "completion_tokens": 355, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-31", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-131", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_science-35", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "cmmlu-college_actuarial_science-38", + "ok": true, + "error": null, + "completion_tokens": 1566, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marketing-165", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marketing-101", + "ok": true, + "error": null, + "completion_tokens": 2266, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-107", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-8", + "ok": true, + "error": null, + "completion_tokens": 6232, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_medical_statistics-86", + "ok": true, + "error": null, + "completion_tokens": 338, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-security_study-30", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-118", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_law-24", + "ok": false, + "error": null, + "completion_tokens": 3082, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-46", + "ok": true, + "error": null, + "completion_tokens": 551, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-anatomy-21", + "ok": false, + "error": null, + "completion_tokens": 8803, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-145", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-arts-91", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-109", + "ok": false, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-44", + "ok": true, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-74", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-63", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-astronomy-146", + "ok": true, + "error": null, + "completion_tokens": 95, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-109", + "ok": true, + "error": null, + "completion_tokens": 418, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-construction_project_management-16", + "ok": true, + "error": null, + "completion_tokens": 288, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-113", + "ok": false, + "error": null, + "completion_tokens": 475, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-102", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-207", + "ok": true, + "error": null, + "completion_tokens": 343, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-economics-19", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-management-164", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-338", + "ok": true, + "error": null, + "completion_tokens": 909, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-7", + "ok": true, + "error": null, + "completion_tokens": 388, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-virology-92", + "ok": true, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-47", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-astronomy-140", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-100", + "ok": true, + "error": null, + "completion_tokens": 758, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-37", + "ok": true, + "error": null, + "completion_tokens": 992, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-191", + "ok": true, + "error": null, + "completion_tokens": 454, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-34", + "ok": true, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-88", + "ok": true, + "error": null, + "completion_tokens": 317, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-public_relations-77", + "ok": false, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-306", + "ok": true, + "error": null, + "completion_tokens": 7821, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-40", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_chemistry-7", + "ok": true, + "error": null, + "completion_tokens": 1899, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-152", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_science-1", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-94", + "ok": true, + "error": null, + "completion_tokens": 71, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-109", + "ok": true, + "error": null, + "completion_tokens": 512, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-164", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sports_science-89", + "ok": false, + "error": null, + "completion_tokens": 1828, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-economics-17", + "ok": true, + "error": null, + "completion_tokens": 517, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-arts-2", + "ok": true, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-genetics-51", + "ok": true, + "error": null, + "completion_tokens": 574, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-agronomy-141", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_science-196", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-genetics-97", + "ok": true, + "error": null, + "completion_tokens": 712, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-202", + "ok": true, + "error": null, + "completion_tokens": 12281, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-management-195", + "ok": true, + "error": null, + "completion_tokens": 1005, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-21", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sociology-49", + "ok": true, + "error": null, + "completion_tokens": 424, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-94", + "ok": true, + "error": null, + "completion_tokens": 1010, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-8", + "ok": true, + "error": null, + "completion_tokens": 780, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-98", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-28", + "ok": true, + "error": null, + "completion_tokens": 269, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-74", + "ok": true, + "error": null, + "completion_tokens": 1097, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-121", + "ok": true, + "error": null, + "completion_tokens": 528, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-120", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-105", + "ok": true, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-35", + "ok": true, + "error": null, + "completion_tokens": 1590, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-51", + "ok": true, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-49", + "ok": true, + "error": null, + "completion_tokens": 2438, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-130", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-virology-140", + "ok": true, + "error": null, + "completion_tokens": 807, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "mmlu_pro": { + "task": "mmlu_pro", + "n": 150, + "hits": 132, + "accuracy_pct": 88.0, + "accuracy_ex_trunc_pct": 91.03, + "truncated": 5, + "completion_tokens_p50": 462, + "completion_tokens_p90": 5861, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 825.6, + "gen_tokens": 410971, + "tok_per_s": 497.8, + "details": [ + { + "id": "mmlupro-5831", + "ok": true, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2299", + "ok": true, + "error": null, + "completion_tokens": 728, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9280", + "ok": true, + "error": null, + "completion_tokens": 5070, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5486", + "ok": true, + "error": null, + "completion_tokens": 1492, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5761", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6589", + "ok": true, + "error": null, + "completion_tokens": 831, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5389", + "ok": true, + "error": null, + "completion_tokens": 290, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5131", + "ok": true, + "error": null, + "completion_tokens": 767, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1677", + "ok": true, + "error": null, + "completion_tokens": 3376, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4388", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-3285", + "ok": true, + "error": null, + "completion_tokens": 1409, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4118", + "ok": true, + "error": null, + "completion_tokens": 2076, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5344", + "ok": true, + "error": null, + "completion_tokens": 609, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11696", + "ok": true, + "error": null, + "completion_tokens": 1716, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-858", + "ok": true, + "error": null, + "completion_tokens": 701, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-214", + "ok": true, + "error": null, + "completion_tokens": 6821, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6609", + "ok": true, + "error": null, + "completion_tokens": 377, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2083", + "ok": true, + "error": null, + "completion_tokens": 578, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9752", + "ok": false, + "error": null, + "completion_tokens": 1367, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7213", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5234", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8570", + "ok": true, + "error": null, + "completion_tokens": 137, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5612", + "ok": true, + "error": null, + "completion_tokens": 889, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2071", + "ok": true, + "error": null, + "completion_tokens": 387, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9842", + "ok": true, + "error": null, + "completion_tokens": 307, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4665", + "ok": true, + "error": null, + "completion_tokens": 907, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5564", + "ok": true, + "error": null, + "completion_tokens": 2369, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6475", + "ok": true, + "error": null, + "completion_tokens": 1636, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2153", + "ok": true, + "error": null, + "completion_tokens": 566, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5812", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3407", + "ok": true, + "error": null, + "completion_tokens": 5562, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2879", + "ok": false, + "error": null, + "completion_tokens": 2146, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8266", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1916", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-3486", + "ok": true, + "error": null, + "completion_tokens": 2054, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11275", + "ok": true, + "error": null, + "completion_tokens": 342, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6040", + "ok": true, + "error": null, + "completion_tokens": 474, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8565", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6848", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8926", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8458", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7854", + "ok": true, + "error": null, + "completion_tokens": 531, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3962", + "ok": true, + "error": null, + "completion_tokens": 1291, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6470", + "ok": true, + "error": null, + "completion_tokens": 500, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3813", + "ok": true, + "error": null, + "completion_tokens": 421, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8026", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11030", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8151", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5286", + "ok": true, + "error": null, + "completion_tokens": 560, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3888", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11676", + "ok": true, + "error": null, + "completion_tokens": 2095, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8762", + "ok": false, + "error": null, + "completion_tokens": 1202, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-12160", + "ok": true, + "error": null, + "completion_tokens": 529, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4022", + "ok": true, + "error": null, + "completion_tokens": 790, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10092", + "ok": true, + "error": null, + "completion_tokens": 720, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7364", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6001", + "ok": true, + "error": null, + "completion_tokens": 453, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10873", + "ok": true, + "error": null, + "completion_tokens": 447, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9777", + "ok": true, + "error": null, + "completion_tokens": 7222, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8723", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4595", + "ok": true, + "error": null, + "completion_tokens": 546, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8423", + "ok": true, + "error": null, + "completion_tokens": 8068, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11051", + "ok": false, + "error": null, + "completion_tokens": 137, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-836", + "ok": true, + "error": null, + "completion_tokens": 363, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11297", + "ok": true, + "error": null, + "completion_tokens": 29048, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3950", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1562", + "ok": false, + "error": null, + "completion_tokens": 4530, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2041", + "ok": true, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3473", + "ok": true, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11253", + "ok": true, + "error": null, + "completion_tokens": 1138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3858", + "ok": true, + "error": null, + "completion_tokens": 872, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2231", + "ok": true, + "error": null, + "completion_tokens": 462, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2190", + "ok": true, + "error": null, + "completion_tokens": 612, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6930", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6558", + "ok": true, + "error": null, + "completion_tokens": 275, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5371", + "ok": true, + "error": null, + "completion_tokens": 11358, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1766", + "ok": true, + "error": null, + "completion_tokens": 1161, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6045", + "ok": true, + "error": null, + "completion_tokens": 4335, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2298", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6400", + "ok": true, + "error": null, + "completion_tokens": 69, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6499", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7936", + "ok": true, + "error": null, + "completion_tokens": 459, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8566", + "ok": true, + "error": null, + "completion_tokens": 90, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11633", + "ok": true, + "error": null, + "completion_tokens": 418, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9843", + "ok": false, + "error": null, + "completion_tokens": 4270, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11758", + "ok": false, + "error": null, + "completion_tokens": 12698, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2843", + "ok": true, + "error": null, + "completion_tokens": 2722, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7483", + "ok": true, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11379", + "ok": true, + "error": null, + "completion_tokens": 873, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4866", + "ok": true, + "error": null, + "completion_tokens": 494, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-818", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2785", + "ok": true, + "error": null, + "completion_tokens": 1781, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8119", + "ok": true, + "error": null, + "completion_tokens": 441, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2214", + "ok": true, + "error": null, + "completion_tokens": 2584, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5774", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-6049", + "ok": false, + "error": null, + "completion_tokens": 2962, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7219", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1660", + "ok": false, + "error": null, + "completion_tokens": 448, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4767", + "ok": false, + "error": null, + "completion_tokens": 14713, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5964", + "ok": false, + "error": null, + "completion_tokens": 10492, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2743", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4062", + "ok": true, + "error": null, + "completion_tokens": 403, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7368", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-827", + "ok": false, + "error": null, + "completion_tokens": 527, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11877", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-7557", + "ok": true, + "error": null, + "completion_tokens": 425, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9998", + "ok": true, + "error": null, + "completion_tokens": 373, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10098", + "ok": false, + "error": null, + "completion_tokens": 9722, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1744", + "ok": true, + "error": null, + "completion_tokens": 3910, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9210", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11248", + "ok": true, + "error": null, + "completion_tokens": 88, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11187", + "ok": true, + "error": null, + "completion_tokens": 237, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2799", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4479", + "ok": true, + "error": null, + "completion_tokens": 1293, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10223", + "ok": true, + "error": null, + "completion_tokens": 1069, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5698", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8291", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3516", + "ok": true, + "error": null, + "completion_tokens": 277, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5466", + "ok": true, + "error": null, + "completion_tokens": 577, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4851", + "ok": true, + "error": null, + "completion_tokens": 553, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-733", + "ok": true, + "error": null, + "completion_tokens": 3847, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6995", + "ok": true, + "error": null, + "completion_tokens": 332, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2573", + "ok": true, + "error": null, + "completion_tokens": 1150, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8481", + "ok": true, + "error": null, + "completion_tokens": 208, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5938", + "ok": true, + "error": null, + "completion_tokens": 289, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10708", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8476", + "ok": true, + "error": null, + "completion_tokens": 397, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7990", + "ok": true, + "error": null, + "completion_tokens": 331, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2409", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11551", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-674", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1978", + "ok": true, + "error": null, + "completion_tokens": 8654, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10420", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3445", + "ok": true, + "error": null, + "completion_tokens": 4583, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9488", + "ok": true, + "error": null, + "completion_tokens": 343, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4993", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4316", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4730", + "ok": true, + "error": null, + "completion_tokens": 274, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11570", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2149", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8322", + "ok": true, + "error": null, + "completion_tokens": 826, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10993", + "ok": true, + "error": null, + "completion_tokens": 5098, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4588", + "ok": true, + "error": null, + "completion_tokens": 5861, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3258", + "ok": true, + "error": null, + "completion_tokens": 591, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8629", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-637", + "ok": true, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4257", + "ok": true, + "error": null, + "completion_tokens": 460, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9719", + "ok": true, + "error": null, + "completion_tokens": 805, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11714", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-9678", + "ok": true, + "error": null, + "completion_tokens": 5106, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "ceval": { + "task": "ceval", + "n": 150, + "hits": 127, + "accuracy_pct": 84.67, + "accuracy_ex_trunc_pct": 85.81, + "truncated": 2, + "completion_tokens_p50": 435, + "completion_tokens_p90": 5504, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 686.8, + "gen_tokens": 307905, + "tok_per_s": 448.3, + "details": [ + { + "id": "ceval-high_school_chinese-1", + "ok": false, + "error": null, + "completion_tokens": 7078, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-10", + "ok": true, + "error": null, + "completion_tokens": 709, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-tax_accountant-41", + "ok": false, + "error": null, + "completion_tokens": 16733, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_physics-3", + "ok": true, + "error": null, + "completion_tokens": 252, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-21", + "ok": false, + "error": null, + "completion_tokens": 987, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-4", + "ok": true, + "error": null, + "completion_tokens": 679, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_history-21", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-2", + "ok": true, + "error": null, + "completion_tokens": 308, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_programming-6", + "ok": true, + "error": null, + "completion_tokens": 454, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-12", + "ok": true, + "error": null, + "completion_tokens": 12366, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-19", + "ok": true, + "error": null, + "completion_tokens": 331, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_chemistry-9", + "ok": true, + "error": null, + "completion_tokens": 2040, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-plant_protection-15", + "ok": true, + "error": null, + "completion_tokens": 258, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-22", + "ok": false, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-plant_protection-16", + "ok": true, + "error": null, + "completion_tokens": 80, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-business_administration-11", + "ok": true, + "error": null, + "completion_tokens": 702, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-fire_engineer-18", + "ok": true, + "error": null, + "completion_tokens": 224, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-24", + "ok": true, + "error": null, + "completion_tokens": 527, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-29", + "ok": true, + "error": null, + "completion_tokens": 1631, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-2", + "ok": true, + "error": null, + "completion_tokens": 782, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-11", + "ok": false, + "error": null, + "completion_tokens": 2037, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-18", + "ok": true, + "error": null, + "completion_tokens": 611, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-0", + "ok": false, + "error": null, + "completion_tokens": 1185, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-14", + "ok": true, + "error": null, + "completion_tokens": 325, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-business_administration-27", + "ok": true, + "error": null, + "completion_tokens": 1586, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-art_studies-19", + "ok": true, + "error": null, + "completion_tokens": 2595, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-21", + "ok": true, + "error": null, + "completion_tokens": 5504, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-tax_accountant-33", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-20", + "ok": true, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-legal_professional-10", + "ok": true, + "error": null, + "completion_tokens": 565, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_programming-19", + "ok": true, + "error": null, + "completion_tokens": 111, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-computer_network-7", + "ok": true, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-41", + "ok": true, + "error": null, + "completion_tokens": 509, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-44", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-9", + "ok": true, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-28", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-39", + "ok": true, + "error": null, + "completion_tokens": 1411, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-1", + "ok": false, + "error": null, + "completion_tokens": 1302, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-business_administration-24", + "ok": true, + "error": null, + "completion_tokens": 90, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-legal_professional-5", + "ok": true, + "error": null, + "completion_tokens": 7404, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-computer_architecture-15", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-39", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-law-23", + "ok": true, + "error": null, + "completion_tokens": 502, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-7", + "ok": true, + "error": null, + "completion_tokens": 703, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-17", + "ok": true, + "error": null, + "completion_tokens": 1526, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_chemistry-6", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-15", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-4", + "ok": false, + "error": null, + "completion_tokens": 17719, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-5", + "ok": true, + "error": null, + "completion_tokens": 1080, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-50", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_politics-11", + "ok": true, + "error": null, + "completion_tokens": 507, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-tax_accountant-32", + "ok": true, + "error": null, + "completion_tokens": 2394, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-16", + "ok": true, + "error": null, + "completion_tokens": 456, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-fire_engineer-0", + "ok": true, + "error": null, + "completion_tokens": 711, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-sports_science-9", + "ok": true, + "error": null, + "completion_tokens": 335, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-art_studies-15", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-2", + "ok": false, + "error": null, + "completion_tokens": 7647, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-fire_engineer-7", + "ok": false, + "error": null, + "completion_tokens": 3154, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-plant_protection-11", + "ok": true, + "error": null, + "completion_tokens": 702, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-operating_system-18", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-4", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_programming-17", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_programming-1", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-business_administration-32", + "ok": true, + "error": null, + "completion_tokens": 156, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-33", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-education_science-25", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-40", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-legal_professional-7", + "ok": true, + "error": null, + "completion_tokens": 5125, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-5", + "ok": true, + "error": null, + "completion_tokens": 1394, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-fire_engineer-12", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-4", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-38", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "ceval-electrical_engineer-22", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-0", + "ok": true, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-15", + "ok": false, + "error": null, + "completion_tokens": 3296, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-45", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-6", + "ok": true, + "error": null, + "completion_tokens": 416, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-legal_professional-20", + "ok": true, + "error": null, + "completion_tokens": 3290, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-basic_medicine-2", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-17", + "ok": true, + "error": null, + "completion_tokens": 435, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-16", + "ok": false, + "error": null, + "completion_tokens": 622, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-7", + "ok": true, + "error": null, + "completion_tokens": 537, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_biology-16", + "ok": false, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-10", + "ok": true, + "error": null, + "completion_tokens": 301, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-32", + "ok": true, + "error": null, + "completion_tokens": 4837, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-0", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-12", + "ok": false, + "error": null, + "completion_tokens": 5182, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-14", + "ok": true, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-26", + "ok": false, + "error": null, + "completion_tokens": 12536, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-3", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-3", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-computer_network-13", + "ok": true, + "error": null, + "completion_tokens": 64, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-sports_science-5", + "ok": true, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-27", + "ok": true, + "error": null, + "completion_tokens": 911, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-fire_engineer-21", + "ok": true, + "error": null, + "completion_tokens": 11287, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-10", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-16", + "ok": true, + "error": null, + "completion_tokens": 10215, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-49", + "ok": true, + "error": null, + "completion_tokens": 611, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-13", + "ok": false, + "error": null, + "completion_tokens": 3619, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-business_administration-5", + "ok": true, + "error": null, + "completion_tokens": 9942, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-2", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-27", + "ok": false, + "error": null, + "completion_tokens": 5582, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_biology-8", + "ok": true, + "error": null, + "completion_tokens": 613, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_physics-14", + "ok": true, + "error": null, + "completion_tokens": 878, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_history-14", + "ok": true, + "error": null, + "completion_tokens": 205, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-31", + "ok": true, + "error": null, + "completion_tokens": 597, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-basic_medicine-13", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-13", + "ok": true, + "error": null, + "completion_tokens": 5337, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_programming-12", + "ok": true, + "error": null, + "completion_tokens": 542, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-44", + "ok": false, + "error": null, + "completion_tokens": 8661, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-3", + "ok": true, + "error": null, + "completion_tokens": 1745, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-13", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "ceval-middle_school_mathematics-6", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_politics-19", + "ok": true, + "error": null, + "completion_tokens": 602, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-environmental_impact_assessment_engineer-27", + "ok": false, + "error": null, + "completion_tokens": 4049, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-law-10", + "ok": true, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-law-0", + "ok": true, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-3", + "ok": true, + "error": null, + "completion_tokens": 321, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-6", + "ok": true, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_chinese-13", + "ok": true, + "error": null, + "completion_tokens": 638, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-43", + "ok": false, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-42", + "ok": true, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-13", + "ok": true, + "error": null, + "completion_tokens": 580, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-discrete_mathematics-1", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-11", + "ok": true, + "error": null, + "completion_tokens": 1181, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-plant_protection-2", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-1", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_physics-16", + "ok": true, + "error": null, + "completion_tokens": 6205, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-11", + "ok": true, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-5", + "ok": true, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-40", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-6", + "ok": true, + "error": null, + "completion_tokens": 347, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_mathematics-10", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-10", + "ok": true, + "error": null, + "completion_tokens": 1658, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-education_science-17", + "ok": true, + "error": null, + "completion_tokens": 1735, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-tax_accountant-11", + "ok": true, + "error": null, + "completion_tokens": 614, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-16", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-education_science-1", + "ok": false, + "error": null, + "completion_tokens": 2888, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_physics-0", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-37", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-9", + "ok": true, + "error": null, + "completion_tokens": 84, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 823, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-5", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-18", + "ok": true, + "error": null, + "completion_tokens": 465, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-30", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-21", + "ok": true, + "error": null, + "completion_tokens": 454, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-42", + "ok": true, + "error": null, + "completion_tokens": 569, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-5", + "ok": true, + "error": null, + "completion_tokens": 1371, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_chemistry-4", + "ok": true, + "error": null, + "completion_tokens": 307, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "arc": { + "task": "arc", + "n": 150, + "hits": 144, + "accuracy_pct": 96.0, + "accuracy_ex_trunc_pct": 96.0, + "truncated": 0, + "completion_tokens_p50": 135, + "completion_tokens_p90": 323, + "completion_tokens_max": 7550, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 127.2, + "gen_tokens": 45568, + "tok_per_s": 358.3, + "details": [ + { + "id": "arc-Mercury_415084", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7249708", + "ok": true, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_4_24", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7056823", + "ok": true, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415534", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405827", + "ok": false, + "error": null, + "completion_tokens": 7550, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7263183", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7106960", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_7", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7182245", + "ok": true, + "error": null, + "completion_tokens": 2752, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416177", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-ACTAAP_2011_5_8", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-TAKS_2009_8_36", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_401699", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7107240", + "ok": true, + "error": null, + "completion_tokens": 72, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_4_26", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402638", + "ok": true, + "error": null, + "completion_tokens": 671, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7026723", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415422", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-LEAP_2000_8_2", + "ok": true, + "error": null, + "completion_tokens": 257, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2006_9_4", + "ok": true, + "error": null, + "completion_tokens": 284, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401272", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416169", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7116235", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7109463", + "ok": true, + "error": null, + "completion_tokens": 83, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7032428", + "ok": true, + "error": null, + "completion_tokens": 7156, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-AIMS_2008_8_11", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_410334", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_36", + "ok": true, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415719", + "ok": true, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_17", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402256", + "ok": true, + "error": null, + "completion_tokens": 205, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_400443", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7133245", + "ok": true, + "error": null, + "completion_tokens": 351, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413458", + "ok": true, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401372", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401116", + "ok": true, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-LEAP__5_10312", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-LEAP_2007_4_10280", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_416683", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7190120", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401295", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_4_28", + "ok": true, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7223493", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7018410", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_1998_8_24", + "ok": true, + "error": null, + "completion_tokens": 309, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_31", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405501", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408030", + "ok": true, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_401684", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7041668", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7205870", + "ok": true, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7210140", + "ok": true, + "error": null, + "completion_tokens": 275, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_405873", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400052", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7018270", + "ok": true, + "error": null, + "completion_tokens": 370, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_406550", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7150728", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400839", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_30", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400383", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7236740", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-CSZ_2005_5_CSZ10021", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7093958", + "ok": true, + "error": null, + "completion_tokens": 162, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7026355", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408324", + "ok": true, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-LEAP__7_10348", + "ok": true, + "error": null, + "completion_tokens": 84, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402629", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400046", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7180810", + "ok": true, + "error": null, + "completion_tokens": 337, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_180005", + "ok": true, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-AIMS_2009_8_14", + "ok": true, + "error": null, + "completion_tokens": 328, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MDSA_2007_5_16", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_LBS10662", + "ok": true, + "error": null, + "completion_tokens": 124, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_23", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2012_8_19", + "ok": false, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7041650", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416181", + "ok": true, + "error": null, + "completion_tokens": 230, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-TIMSS_1995_8_J6", + "ok": true, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2014_5_13", + "ok": true, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7106750", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_1999_4_23", + "ok": true, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MEA_2013_5_12", + "ok": true, + "error": null, + "completion_tokens": 90, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_8_42", + "ok": false, + "error": null, + "completion_tokens": 431, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408367", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7250058", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_414041", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401290", + "ok": true, + "error": null, + "completion_tokens": 83, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_409295", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7211680", + "ok": true, + "error": null, + "completion_tokens": 224, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_37", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413439", + "ok": true, + "error": null, + "completion_tokens": 186, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_32", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7216423", + "ok": true, + "error": null, + "completion_tokens": 257, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-ACTAAP_2015_7_9", + "ok": true, + "error": null, + "completion_tokens": 222, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7230405", + "ok": true, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7025008", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_416536", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-TIMSS_2007_8_pg60", + "ok": true, + "error": null, + "completion_tokens": 145, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MDSA_2009_4_34", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_406833", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_409647", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_184363", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7107170", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416104", + "ok": true, + "error": null, + "completion_tokens": 98, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_10-v1", + "ok": true, + "error": null, + "completion_tokens": 247, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7202843", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7029803", + "ok": true, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7164623", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_26", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7263060", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7268328", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402282", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7029558", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_4_21", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7186358", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-LEAP_2011_8_10434", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2012_8_23651", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_407125", + "ok": true, + "error": null, + "completion_tokens": 246, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7268258", + "ok": true, + "error": null, + "completion_tokens": 1322, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7027405", + "ok": true, + "error": null, + "completion_tokens": 119, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2004_8_7", + "ok": true, + "error": null, + "completion_tokens": 488, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_191625", + "ok": true, + "error": null, + "completion_tokens": 3437, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_6", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7219678", + "ok": true, + "error": null, + "completion_tokens": 262, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401678", + "ok": true, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_401312", + "ok": true, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_8_42", + "ok": false, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7030468", + "ok": true, + "error": null, + "completion_tokens": 431, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_175385", + "ok": true, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_175560", + "ok": true, + "error": null, + "completion_tokens": 54, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_5", + "ok": true, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_409009", + "ok": true, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7128923", + "ok": false, + "error": null, + "completion_tokens": 375, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_8_9", + "ok": false, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7042945", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_8_47", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7185343", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7218400", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7221988", + "ok": true, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_405950", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7222530", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7102323", + "ok": true, + "error": null, + "completion_tokens": 54, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MDSA_2009_5_39", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_1999_8_5", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400661", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7081603", + "ok": true, + "error": null, + "completion_tokens": 276, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-VASoL_2008_3_25", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413628", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_406427", + "ok": true, + "error": null, + "completion_tokens": 124, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "truthfulqa": { + "task": "truthfulqa", + "n": 150, + "hits": 132, + "accuracy_pct": 88.0, + "accuracy_ex_trunc_pct": 90.41, + "truncated": 4, + "completion_tokens_p50": 250, + "completion_tokens_p90": 2269, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 655.4, + "gen_tokens": 268199, + "tok_per_s": 409.2, + "details": [ + { + "id": "tqa-210", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "tqa-76", + "ok": true, + "error": null, + "completion_tokens": 2881, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-461", + "ok": true, + "error": null, + "completion_tokens": 562, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-218", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-254", + "ok": true, + "error": null, + "completion_tokens": 2084, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-635", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-193", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-148", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-400", + "ok": false, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-766", + "ok": true, + "error": null, + "completion_tokens": 187, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-240", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "tqa-50", + "ok": true, + "error": null, + "completion_tokens": 302, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-83", + "ok": true, + "error": null, + "completion_tokens": 655, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-381", + "ok": true, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-325", + "ok": true, + "error": null, + "completion_tokens": 566, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-655", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-721", + "ok": true, + "error": null, + "completion_tokens": 5458, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-391", + "ok": true, + "error": null, + "completion_tokens": 422, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-401", + "ok": true, + "error": null, + "completion_tokens": 630, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-625", + "ok": true, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-80", + "ok": false, + "error": null, + "completion_tokens": 218, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-439", + "ok": true, + "error": null, + "completion_tokens": 469, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-66", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "tqa-155", + "ok": true, + "error": null, + "completion_tokens": 309, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-516", + "ok": true, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-522", + "ok": true, + "error": null, + "completion_tokens": 1000, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-47", + "ok": false, + "error": null, + "completion_tokens": 12798, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-343", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-126", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-77", + "ok": true, + "error": null, + "completion_tokens": 359, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-613", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-590", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-674", + "ok": true, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-438", + "ok": true, + "error": null, + "completion_tokens": 2743, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-503", + "ok": true, + "error": null, + "completion_tokens": 427, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-639", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-111", + "ok": true, + "error": null, + "completion_tokens": 1186, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-289", + "ok": true, + "error": null, + "completion_tokens": 647, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-705", + "ok": true, + "error": null, + "completion_tokens": 739, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-373", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-363", + "ok": true, + "error": null, + "completion_tokens": 455, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-792", + "ok": true, + "error": null, + "completion_tokens": 259, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-113", + "ok": false, + "error": null, + "completion_tokens": 881, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-74", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-634", + "ok": true, + "error": null, + "completion_tokens": 625, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-804", + "ok": true, + "error": null, + "completion_tokens": 2135, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-626", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-676", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-174", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-455", + "ok": true, + "error": null, + "completion_tokens": 695, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-725", + "ok": true, + "error": null, + "completion_tokens": 473, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-296", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-535", + "ok": true, + "error": null, + "completion_tokens": 654, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-172", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-23", + "ok": true, + "error": null, + "completion_tokens": 172, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-43", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-38", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-97", + "ok": false, + "error": null, + "completion_tokens": 8073, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-657", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-814", + "ok": true, + "error": null, + "completion_tokens": 226, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-346", + "ok": true, + "error": null, + "completion_tokens": 407, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-365", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-384", + "ok": true, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-677", + "ok": true, + "error": null, + "completion_tokens": 2042, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-486", + "ok": true, + "error": null, + "completion_tokens": 486, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-55", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-680", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-510", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-128", + "ok": true, + "error": null, + "completion_tokens": 318, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-805", + "ok": true, + "error": null, + "completion_tokens": 3857, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-394", + "ok": false, + "error": null, + "completion_tokens": 290, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-435", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-285", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-687", + "ok": true, + "error": null, + "completion_tokens": 203, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-588", + "ok": true, + "error": null, + "completion_tokens": 511, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-100", + "ok": true, + "error": null, + "completion_tokens": 250, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-399", + "ok": true, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-266", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-693", + "ok": true, + "error": null, + "completion_tokens": 264, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-532", + "ok": false, + "error": null, + "completion_tokens": 4141, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-558", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-319", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-459", + "ok": true, + "error": null, + "completion_tokens": 393, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-675", + "ok": true, + "error": null, + "completion_tokens": 276, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-129", + "ok": true, + "error": null, + "completion_tokens": 837, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-331", + "ok": false, + "error": null, + "completion_tokens": 7818, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-12", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-164", + "ok": false, + "error": null, + "completion_tokens": 15645, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-306", + "ok": true, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-275", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-418", + "ok": true, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-264", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-94", + "ok": true, + "error": null, + "completion_tokens": 1166, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-389", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-427", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-269", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-643", + "ok": true, + "error": null, + "completion_tokens": 759, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-7", + "ok": true, + "error": null, + "completion_tokens": 5130, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-419", + "ok": true, + "error": null, + "completion_tokens": 191, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-616", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-699", + "ok": true, + "error": null, + "completion_tokens": 178, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-473", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-523", + "ok": true, + "error": null, + "completion_tokens": 385, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-214", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-376", + "ok": true, + "error": null, + "completion_tokens": 568, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-123", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-200", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-493", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-474", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-387", + "ok": true, + "error": null, + "completion_tokens": 71, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-746", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-790", + "ok": true, + "error": null, + "completion_tokens": 237, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-181", + "ok": true, + "error": null, + "completion_tokens": 1671, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-413", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-349", + "ok": true, + "error": null, + "completion_tokens": 2269, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-434", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-601", + "ok": true, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-489", + "ok": true, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-286", + "ok": false, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-488", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-759", + "ok": true, + "error": null, + "completion_tokens": 288, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-273", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-59", + "ok": true, + "error": null, + "completion_tokens": 407, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-315", + "ok": true, + "error": null, + "completion_tokens": 765, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-425", + "ok": true, + "error": null, + "completion_tokens": 512, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-58", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-691", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-497", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-515", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-398", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-670", + "ok": true, + "error": null, + "completion_tokens": 16629, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-170", + "ok": false, + "error": null, + "completion_tokens": 570, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-334", + "ok": false, + "error": null, + "completion_tokens": 872, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-307", + "ok": false, + "error": null, + "completion_tokens": 740, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-495", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-597", + "ok": true, + "error": null, + "completion_tokens": 1977, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-775", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-186", + "ok": true, + "error": null, + "completion_tokens": 398, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-733", + "ok": true, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-308", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "tqa-5", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-231", + "ok": true, + "error": null, + "completion_tokens": 1047, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-356", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-810", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-270", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-13", + "ok": false, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-114", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-4", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-417", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-750", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "gsm8k": { + "task": "gsm8k", + "n": 100, + "hits": 99, + "accuracy_pct": 99.0, + "accuracy_ex_trunc_pct": 99.0, + "truncated": 0, + "completion_tokens_p50": 220, + "completion_tokens_p90": 487, + "completion_tokens_max": 3411, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 74.0, + "gen_tokens": 36114, + "tok_per_s": 487.9, + "details": [ + { + "id": "gsm8k-144", + "ok": true, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-627", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-195", + "ok": true, + "error": null, + "completion_tokens": 268, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-201", + "ok": true, + "error": null, + "completion_tokens": 1881, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-730", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-166", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1141", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-832", + "ok": true, + "error": null, + "completion_tokens": 415, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-53", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1042", + "ok": false, + "error": null, + "completion_tokens": 2508, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-609", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-318", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-892", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-204", + "ok": true, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1120", + "ok": true, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1040", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-556", + "ok": true, + "error": null, + "completion_tokens": 250, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1015", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-206", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1058", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1044", + "ok": true, + "error": null, + "completion_tokens": 321, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-131", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1154", + "ok": true, + "error": null, + "completion_tokens": 124, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-158", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1149", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1156", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-60", + "ok": true, + "error": null, + "completion_tokens": 349, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-918", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1010", + "ok": true, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1032", + "ok": true, + "error": null, + "completion_tokens": 268, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1053", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-577", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-935", + "ok": true, + "error": null, + "completion_tokens": 317, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-692", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-153", + "ok": true, + "error": null, + "completion_tokens": 534, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-40", + "ok": true, + "error": null, + "completion_tokens": 940, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-613", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-875", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-189", + "ok": true, + "error": null, + "completion_tokens": 755, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-800", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-169", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1169", + "ok": true, + "error": null, + "completion_tokens": 899, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-211", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1183", + "ok": true, + "error": null, + "completion_tokens": 3411, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-193", + "ok": true, + "error": null, + "completion_tokens": 288, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-230", + "ok": true, + "error": null, + "completion_tokens": 329, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-787", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-83", + "ok": true, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1318", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-733", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1205", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-88", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-509", + "ok": true, + "error": null, + "completion_tokens": 309, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-311", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-874", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-287", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1055", + "ok": true, + "error": null, + "completion_tokens": 274, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1246", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-975", + "ok": true, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-280", + "ok": true, + "error": null, + "completion_tokens": 124, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-541", + "ok": true, + "error": null, + "completion_tokens": 339, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-974", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-827", + "ok": true, + "error": null, + "completion_tokens": 487, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-125", + "ok": true, + "error": null, + "completion_tokens": 404, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-453", + "ok": true, + "error": null, + "completion_tokens": 192, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-942", + "ok": true, + "error": null, + "completion_tokens": 282, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1199", + "ok": true, + "error": null, + "completion_tokens": 318, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-884", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-855", + "ok": true, + "error": null, + "completion_tokens": 408, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1270", + "ok": true, + "error": null, + "completion_tokens": 457, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-349", + "ok": true, + "error": null, + "completion_tokens": 200, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1231", + "ok": true, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1106", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-803", + "ok": true, + "error": null, + "completion_tokens": 237, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-922", + "ok": true, + "error": null, + "completion_tokens": 805, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-152", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-920", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-207", + "ok": true, + "error": null, + "completion_tokens": 212, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-297", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-786", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-101", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1301", + "ok": true, + "error": null, + "completion_tokens": 335, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-27", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1005", + "ok": true, + "error": null, + "completion_tokens": 1015, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-659", + "ok": true, + "error": null, + "completion_tokens": 261, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-340", + "ok": true, + "error": null, + "completion_tokens": 2695, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-288", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-763", + "ok": true, + "error": null, + "completion_tokens": 315, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-840", + "ok": true, + "error": null, + "completion_tokens": 341, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-742", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-222", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-468", + "ok": true, + "error": null, + "completion_tokens": 376, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-220", + "ok": true, + "error": null, + "completion_tokens": 119, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-662", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-269", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1193", + "ok": true, + "error": null, + "completion_tokens": 420, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-270", + "ok": true, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-114", + "ok": true, + "error": null, + "completion_tokens": 374, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-728", + "ok": true, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-396", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "math500": { + "task": "math500", + "n": 100, + "hits": 75, + "accuracy_pct": 75.0, + "accuracy_ex_trunc_pct": 75.0, + "truncated": 0, + "completion_tokens_p50": 463, + "completion_tokens_p90": 4228, + "completion_tokens_max": 17878, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 324.7, + "gen_tokens": 139621, + "tok_per_s": 430.0, + "details": [ + { + "id": "math-test/algebra/1553.json", + "ok": true, + "error": null, + "completion_tokens": 505, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/538.json", + "ok": true, + "error": null, + "completion_tokens": 641, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/691.json", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/1090.json", + "ok": true, + "error": null, + "completion_tokens": 684, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/116.json", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1255.json", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1143.json", + "ok": true, + "error": null, + "completion_tokens": 836, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/precalculus/1201.json", + "ok": true, + "error": null, + "completion_tokens": 5597, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/964.json", + "ok": true, + "error": null, + "completion_tokens": 4468, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/662.json", + "ok": false, + "error": null, + "completion_tokens": 2075, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/153.json", + "ok": false, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/276.json", + "ok": true, + "error": null, + "completion_tokens": 576, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1840.json", + "ok": true, + "error": null, + "completion_tokens": 1421, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1072.json", + "ok": false, + "error": null, + "completion_tokens": 607, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1907.json", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/567.json", + "ok": true, + "error": null, + "completion_tokens": 480, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1922.json", + "ok": true, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1779.json", + "ok": true, + "error": null, + "completion_tokens": 5967, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/951.json", + "ok": true, + "error": null, + "completion_tokens": 325, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1197.json", + "ok": false, + "error": null, + "completion_tokens": 3713, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/853.json", + "ok": true, + "error": null, + "completion_tokens": 889, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/516.json", + "ok": false, + "error": null, + "completion_tokens": 1090, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1114.json", + "ok": false, + "error": null, + "completion_tokens": 258, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/1065.json", + "ok": true, + "error": null, + "completion_tokens": 590, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1232.json", + "ok": false, + "error": null, + "completion_tokens": 335, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/769.json", + "ok": true, + "error": null, + "completion_tokens": 754, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1791.json", + "ok": false, + "error": null, + "completion_tokens": 3529, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/246.json", + "ok": true, + "error": null, + "completion_tokens": 387, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1166.json", + "ok": true, + "error": null, + "completion_tokens": 5023, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1410.json", + "ok": false, + "error": null, + "completion_tokens": 464, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1686.json", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/754.json", + "ok": false, + "error": null, + "completion_tokens": 1083, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/477.json", + "ok": true, + "error": null, + "completion_tokens": 4537, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2680.json", + "ok": true, + "error": null, + "completion_tokens": 274, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/23957.json", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1865.json", + "ok": true, + "error": null, + "completion_tokens": 1116, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1014.json", + "ok": true, + "error": null, + "completion_tokens": 380, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/431.json", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1425.json", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2584.json", + "ok": false, + "error": null, + "completion_tokens": 328, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2470.json", + "ok": true, + "error": null, + "completion_tokens": 728, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/547.json", + "ok": true, + "error": null, + "completion_tokens": 4712, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/precalculus/1105.json", + "ok": false, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/347.json", + "ok": true, + "error": null, + "completion_tokens": 351, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/1257.json", + "ok": true, + "error": null, + "completion_tokens": 623, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1126.json", + "ok": true, + "error": null, + "completion_tokens": 650, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/239.json", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1961.json", + "ok": false, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1388.json", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1078.json", + "ok": true, + "error": null, + "completion_tokens": 759, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2157.json", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/339.json", + "ok": true, + "error": null, + "completion_tokens": 925, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1761.json", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1960.json", + "ok": true, + "error": null, + "completion_tokens": 302, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/2086.json", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1462.json", + "ok": true, + "error": null, + "completion_tokens": 3034, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1060.json", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/1055.json", + "ok": true, + "error": null, + "completion_tokens": 2228, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2517.json", + "ok": false, + "error": null, + "completion_tokens": 854, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/996.json", + "ok": true, + "error": null, + "completion_tokens": 256, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/893.json", + "ok": true, + "error": null, + "completion_tokens": 499, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/51.json", + "ok": false, + "error": null, + "completion_tokens": 321, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/precalculus/1291.json", + "ok": true, + "error": null, + "completion_tokens": 463, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/694.json", + "ok": true, + "error": null, + "completion_tokens": 397, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/711.json", + "ok": false, + "error": null, + "completion_tokens": 17878, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1655.json", + "ok": false, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1936.json", + "ok": true, + "error": null, + "completion_tokens": 186, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1572.json", + "ok": true, + "error": null, + "completion_tokens": 392, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/509.json", + "ok": false, + "error": null, + "completion_tokens": 719, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/864.json", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1405.json", + "ok": true, + "error": null, + "completion_tokens": 1864, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/192.json", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2102.json", + "ok": true, + "error": null, + "completion_tokens": 368, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/843.json", + "ok": true, + "error": null, + "completion_tokens": 378, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/187.json", + "ok": true, + "error": null, + "completion_tokens": 359, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/612.json", + "ok": true, + "error": null, + "completion_tokens": 569, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/615.json", + "ok": true, + "error": null, + "completion_tokens": 5190, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1908.json", + "ok": true, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/928.json", + "ok": true, + "error": null, + "completion_tokens": 1248, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/505.json", + "ok": true, + "error": null, + "completion_tokens": 205, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1930.json", + "ok": false, + "error": null, + "completion_tokens": 8874, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1934.json", + "ok": true, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1139.json", + "ok": true, + "error": null, + "completion_tokens": 12244, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1000.json", + "ok": true, + "error": null, + "completion_tokens": 1113, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/183.json", + "ok": true, + "error": null, + "completion_tokens": 1326, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1275.json", + "ok": true, + "error": null, + "completion_tokens": 431, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2700.json", + "ok": false, + "error": null, + "completion_tokens": 875, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2257.json", + "ok": false, + "error": null, + "completion_tokens": 381, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/14.json", + "ok": false, + "error": null, + "completion_tokens": 2480, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/precalculus/1300.json", + "ok": false, + "error": null, + "completion_tokens": 859, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/515.json", + "ok": true, + "error": null, + "completion_tokens": 379, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1423.json", + "ok": false, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1102.json", + "ok": true, + "error": null, + "completion_tokens": 906, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1646.json", + "ok": true, + "error": null, + "completion_tokens": 4228, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1272.json", + "ok": false, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/25149.json", + "ok": true, + "error": null, + "completion_tokens": 329, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/precalculus/697.json", + "ok": true, + "error": null, + "completion_tokens": 4052, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2036.json", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/2037.json", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/353.json", + "ok": true, + "error": null, + "completion_tokens": 341, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "bbh": { + "task": "bbh", + "n": 150, + "hits": 140, + "accuracy_pct": 93.33, + "accuracy_ex_trunc_pct": 94.56, + "truncated": 3, + "completion_tokens_p50": 442, + "completion_tokens_p90": 6631, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 638.0, + "gen_tokens": 339450, + "tok_per_s": 532.0, + "details": [ + { + "id": "bbh-sports_understanding-101", + "ok": true, + "error": null, + "completion_tokens": 226, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-60", + "ok": true, + "error": null, + "completion_tokens": 451, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-113", + "ok": true, + "error": null, + "completion_tokens": 3152, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-web_of_lies-31", + "ok": true, + "error": null, + "completion_tokens": 656, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-144", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "bbh-hyperbaton-28", + "ok": true, + "error": null, + "completion_tokens": 383, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-116", + "ok": true, + "error": null, + "completion_tokens": 272, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-67", + "ok": true, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-3", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-29", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-192", + "ok": true, + "error": null, + "completion_tokens": 1095, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-196", + "ok": true, + "error": null, + "completion_tokens": 424, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-10", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-83", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-76", + "ok": true, + "error": null, + "completion_tokens": 683, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-212", + "ok": true, + "error": null, + "completion_tokens": 454, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-87", + "ok": true, + "error": null, + "completion_tokens": 541, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-1", + "ok": true, + "error": null, + "completion_tokens": 1720, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-140", + "ok": true, + "error": null, + "completion_tokens": 823, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-8", + "ok": true, + "error": null, + "completion_tokens": 603, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-138", + "ok": true, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-221", + "ok": true, + "error": null, + "completion_tokens": 869, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-causal_judgement-136", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "bbh-penguins_in_a_table-14", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-0", + "ok": true, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-207", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-32", + "ok": true, + "error": null, + "completion_tokens": 401, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-83", + "ok": true, + "error": null, + "completion_tokens": 552, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-220", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-234", + "ok": true, + "error": null, + "completion_tokens": 722, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-ruin_names-61", + "ok": true, + "error": null, + "completion_tokens": 8640, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-21", + "ok": true, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-31", + "ok": true, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-13", + "ok": true, + "error": null, + "completion_tokens": 885, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-211", + "ok": true, + "error": null, + "completion_tokens": 315, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-37", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-225", + "ok": true, + "error": null, + "completion_tokens": 335, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-90", + "ok": true, + "error": null, + "completion_tokens": 384, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-101", + "ok": true, + "error": null, + "completion_tokens": 2701, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-web_of_lies-179", + "ok": true, + "error": null, + "completion_tokens": 469, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-173", + "ok": true, + "error": null, + "completion_tokens": 266, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-105", + "ok": true, + "error": null, + "completion_tokens": 187, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-88", + "ok": true, + "error": null, + "completion_tokens": 623, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-68", + "ok": true, + "error": null, + "completion_tokens": 684, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-185", + "ok": true, + "error": null, + "completion_tokens": 11215, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-227", + "ok": true, + "error": null, + "completion_tokens": 377, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-129", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-238", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-202", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-93", + "ok": true, + "error": null, + "completion_tokens": 525, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-93", + "ok": true, + "error": null, + "completion_tokens": 2382, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-161", + "ok": true, + "error": null, + "completion_tokens": 482, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-sports_understanding-46", + "ok": true, + "error": null, + "completion_tokens": 7271, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-227", + "ok": true, + "error": null, + "completion_tokens": 676, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-158", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-205", + "ok": true, + "error": null, + "completion_tokens": 1507, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-89", + "ok": true, + "error": null, + "completion_tokens": 428, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-177", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-65", + "ok": true, + "error": null, + "completion_tokens": 355, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-72", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-49", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-54", + "ok": true, + "error": null, + "completion_tokens": 1036, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-64", + "ok": true, + "error": null, + "completion_tokens": 523, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-86", + "ok": true, + "error": null, + "completion_tokens": 1607, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-108", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-194", + "ok": true, + "error": null, + "completion_tokens": 645, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-194", + "ok": true, + "error": null, + "completion_tokens": 314, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-dyck_languages-75", + "ok": false, + "error": null, + "completion_tokens": 6631, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-106", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-81", + "ok": true, + "error": null, + "completion_tokens": 24698, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-242", + "ok": true, + "error": null, + "completion_tokens": 212, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-navigate-150", + "ok": true, + "error": null, + "completion_tokens": 445, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-causal_judgement-25", + "ok": true, + "error": null, + "completion_tokens": 449, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-139", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-12", + "ok": true, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-178", + "ok": true, + "error": null, + "completion_tokens": 883, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-160", + "ok": true, + "error": null, + "completion_tokens": 307, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-dyck_languages-115", + "ok": true, + "error": null, + "completion_tokens": 4281, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-44", + "ok": true, + "error": null, + "completion_tokens": 811, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-189", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-102", + "ok": true, + "error": null, + "completion_tokens": 1477, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-38", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-141", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-38", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-168", + "ok": true, + "error": null, + "completion_tokens": 442, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-dyck_languages-212", + "ok": true, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "bbh-navigate-245", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-222", + "ok": true, + "error": null, + "completion_tokens": 3129, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-244", + "ok": true, + "error": null, + "completion_tokens": 293, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-35", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-126", + "ok": false, + "error": null, + "completion_tokens": 778, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-90", + "ok": true, + "error": null, + "completion_tokens": 1248, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-5", + "ok": true, + "error": null, + "completion_tokens": 266, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-sports_understanding-241", + "ok": true, + "error": null, + "completion_tokens": 14487, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-234", + "ok": true, + "error": null, + "completion_tokens": 347, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-233", + "ok": true, + "error": null, + "completion_tokens": 202, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-5", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-causal_judgement-152", + "ok": true, + "error": null, + "completion_tokens": 1335, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-9", + "ok": true, + "error": null, + "completion_tokens": 88, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-209", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-171", + "ok": true, + "error": null, + "completion_tokens": 397, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-navigate-160", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-166", + "ok": false, + "error": null, + "completion_tokens": 14490, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-213", + "ok": true, + "error": null, + "completion_tokens": 1571, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-240", + "ok": true, + "error": null, + "completion_tokens": 1944, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-95", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-227", + "ok": true, + "error": null, + "completion_tokens": 626, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-96", + "ok": true, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-33", + "ok": true, + "error": null, + "completion_tokens": 308, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-4", + "ok": true, + "error": null, + "completion_tokens": 443, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-180", + "ok": true, + "error": null, + "completion_tokens": 1474, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-74", + "ok": true, + "error": null, + "completion_tokens": 406, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-88", + "ok": true, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-10", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-dyck_languages-191", + "ok": true, + "error": null, + "completion_tokens": 6926, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-71", + "ok": true, + "error": null, + "completion_tokens": 1063, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-150", + "ok": false, + "error": null, + "completion_tokens": 11964, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-188", + "ok": false, + "error": null, + "completion_tokens": 2375, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-191", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-web_of_lies-203", + "ok": true, + "error": null, + "completion_tokens": 1122, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-ruin_names-237", + "ok": true, + "error": null, + "completion_tokens": 714, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-24", + "ok": true, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-27", + "ok": true, + "error": null, + "completion_tokens": 426, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-177", + "ok": true, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-192", + "ok": true, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-242", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-dyck_languages-51", + "ok": false, + "error": null, + "completion_tokens": 8677, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-96", + "ok": true, + "error": null, + "completion_tokens": 972, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-100", + "ok": true, + "error": null, + "completion_tokens": 326, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-204", + "ok": true, + "error": null, + "completion_tokens": 245, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-58", + "ok": true, + "error": null, + "completion_tokens": 560, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-157", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-104", + "ok": true, + "error": null, + "completion_tokens": 1041, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-47", + "ok": true, + "error": null, + "completion_tokens": 448, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-150", + "ok": true, + "error": null, + "completion_tokens": 387, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-23", + "ok": true, + "error": null, + "completion_tokens": 339, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-sports_understanding-68", + "ok": true, + "error": null, + "completion_tokens": 720, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-174", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-108", + "ok": true, + "error": null, + "completion_tokens": 2855, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-203", + "ok": false, + "error": null, + "completion_tokens": 3059, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-115", + "ok": true, + "error": null, + "completion_tokens": 367, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-223", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-221", + "ok": true, + "error": null, + "completion_tokens": 12525, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-5", + "ok": true, + "error": null, + "completion_tokens": 375, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-233", + "ok": true, + "error": null, + "completion_tokens": 16535, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-ruin_names-3", + "ok": true, + "error": null, + "completion_tokens": 547, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-119", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-197", + "ok": true, + "error": null, + "completion_tokens": 961, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-103", + "ok": true, + "error": null, + "completion_tokens": 4334, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-causal_judgement-178", + "ok": false, + "error": null, + "completion_tokens": 7115, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "humaneval": { + "task": "humaneval", + "n": 164, + "hits": 143, + "accuracy_pct": 87.2, + "accuracy_ex_trunc_pct": 89.94, + "truncated": 5, + "completion_tokens_p50": 620, + "completion_tokens_p90": 4106, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 780.9, + "gen_tokens": 414122, + "tok_per_s": 530.3, + "details": [ + { + "id": "HumanEval/0", + "ok": true, + "error": null, + "completion_tokens": 670, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/1", + "ok": true, + "error": null, + "completion_tokens": 782, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/2", + "ok": true, + "error": null, + "completion_tokens": 504, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/3", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/4", + "ok": false, + "error": null, + "completion_tokens": 699, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/5", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/6", + "ok": false, + "error": null, + "completion_tokens": 804, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/7", + "ok": true, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/8", + "ok": false, + "error": null, + "completion_tokens": 515, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/9", + "ok": false, + "error": null, + "completion_tokens": 799, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/10", + "ok": true, + "error": null, + "completion_tokens": 1641, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/11", + "ok": true, + "error": null, + "completion_tokens": 2616, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/12", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/13", + "ok": true, + "error": null, + "completion_tokens": 264, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/14", + "ok": false, + "error": null, + "completion_tokens": 1311, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/15", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/16", + "ok": true, + "error": null, + "completion_tokens": 510, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/17", + "ok": true, + "error": null, + "completion_tokens": 1228, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/18", + "ok": true, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/19", + "ok": true, + "error": null, + "completion_tokens": 437, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/20", + "ok": false, + "error": null, + "completion_tokens": 548, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/21", + "ok": false, + "error": null, + "completion_tokens": 560, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/22", + "ok": false, + "error": null, + "completion_tokens": 5013, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/23", + "ok": true, + "error": null, + "completion_tokens": 116, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/24", + "ok": true, + "error": null, + "completion_tokens": 851, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/25", + "ok": false, + "error": null, + "completion_tokens": 1629, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/26", + "ok": false, + "error": null, + "completion_tokens": 430, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/27", + "ok": true, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/28", + "ok": true, + "error": null, + "completion_tokens": 1097, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/29", + "ok": true, + "error": null, + "completion_tokens": 902, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/30", + "ok": true, + "error": null, + "completion_tokens": 224, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/31", + "ok": true, + "error": null, + "completion_tokens": 848, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/32", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/33", + "ok": true, + "error": null, + "completion_tokens": 493, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/34", + "ok": true, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/35", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/36", + "ok": true, + "error": null, + "completion_tokens": 2657, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/37", + "ok": true, + "error": null, + "completion_tokens": 572, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/38", + "ok": false, + "error": null, + "completion_tokens": 351, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/39", + "ok": true, + "error": null, + "completion_tokens": 1108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/40", + "ok": true, + "error": null, + "completion_tokens": 1144, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/41", + "ok": true, + "error": null, + "completion_tokens": 372, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/42", + "ok": true, + "error": null, + "completion_tokens": 222, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/43", + "ok": true, + "error": null, + "completion_tokens": 544, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/44", + "ok": true, + "error": null, + "completion_tokens": 1021, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/45", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/46", + "ok": false, + "error": null, + "completion_tokens": 1269, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/47", + "ok": true, + "error": null, + "completion_tokens": 23918, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/48", + "ok": true, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/49", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/50", + "ok": false, + "error": null, + "completion_tokens": 441, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/51", + "ok": true, + "error": null, + "completion_tokens": 290, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/52", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/53", + "ok": true, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/54", + "ok": true, + "error": null, + "completion_tokens": 509, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/55", + "ok": true, + "error": null, + "completion_tokens": 460, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/56", + "ok": true, + "error": null, + "completion_tokens": 400, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/57", + "ok": true, + "error": null, + "completion_tokens": 401, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/58", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/59", + "ok": true, + "error": null, + "completion_tokens": 638, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/60", + "ok": true, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/61", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/62", + "ok": true, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/63", + "ok": true, + "error": null, + "completion_tokens": 620, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/64", + "ok": true, + "error": null, + "completion_tokens": 589, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/65", + "ok": true, + "error": null, + "completion_tokens": 1968, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/66", + "ok": true, + "error": null, + "completion_tokens": 550, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/67", + "ok": true, + "error": null, + "completion_tokens": 512, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/68", + "ok": true, + "error": null, + "completion_tokens": 866, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/69", + "ok": true, + "error": null, + "completion_tokens": 757, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/70", + "ok": true, + "error": null, + "completion_tokens": 811, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/71", + "ok": true, + "error": null, + "completion_tokens": 1281, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/72", + "ok": true, + "error": null, + "completion_tokens": 394, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/73", + "ok": true, + "error": null, + "completion_tokens": 499, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/74", + "ok": true, + "error": null, + "completion_tokens": 452, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/75", + "ok": true, + "error": null, + "completion_tokens": 6959, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/76", + "ok": true, + "error": null, + "completion_tokens": 11556, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/77", + "ok": true, + "error": null, + "completion_tokens": 747, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/78", + "ok": true, + "error": null, + "completion_tokens": 520, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/79", + "ok": true, + "error": null, + "completion_tokens": 465, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/80", + "ok": true, + "error": null, + "completion_tokens": 331, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/81", + "ok": true, + "error": null, + "completion_tokens": 1633, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/82", + "ok": true, + "error": null, + "completion_tokens": 436, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/83", + "ok": true, + "error": null, + "completion_tokens": 929, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/84", + "ok": true, + "error": null, + "completion_tokens": 312, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/85", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/86", + "ok": true, + "error": null, + "completion_tokens": 1613, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/87", + "ok": true, + "error": null, + "completion_tokens": 1629, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/88", + "ok": true, + "error": null, + "completion_tokens": 453, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/89", + "ok": true, + "error": null, + "completion_tokens": 721, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/90", + "ok": true, + "error": null, + "completion_tokens": 546, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/91", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/92", + "ok": true, + "error": null, + "completion_tokens": 1131, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/93", + "ok": true, + "error": null, + "completion_tokens": 1495, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/94", + "ok": true, + "error": null, + "completion_tokens": 5245, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/95", + "ok": true, + "error": null, + "completion_tokens": 549, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/96", + "ok": true, + "error": null, + "completion_tokens": 624, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/97", + "ok": true, + "error": null, + "completion_tokens": 303, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/98", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/99", + "ok": true, + "error": null, + "completion_tokens": 2200, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/100", + "ok": true, + "error": null, + "completion_tokens": 484, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/101", + "ok": true, + "error": null, + "completion_tokens": 293, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/102", + "ok": true, + "error": null, + "completion_tokens": 4106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/103", + "ok": true, + "error": null, + "completion_tokens": 2036, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/104", + "ok": true, + "error": null, + "completion_tokens": 448, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/105", + "ok": true, + "error": null, + "completion_tokens": 1945, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/106", + "ok": true, + "error": null, + "completion_tokens": 901, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/107", + "ok": true, + "error": null, + "completion_tokens": 630, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/108", + "ok": true, + "error": null, + "completion_tokens": 820, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/109", + "ok": true, + "error": null, + "completion_tokens": 1091, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/110", + "ok": true, + "error": null, + "completion_tokens": 646, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/111", + "ok": true, + "error": null, + "completion_tokens": 703, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/112", + "ok": true, + "error": null, + "completion_tokens": 350, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/113", + "ok": true, + "error": null, + "completion_tokens": 903, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/114", + "ok": true, + "error": null, + "completion_tokens": 458, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/115", + "ok": true, + "error": null, + "completion_tokens": 441, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/116", + "ok": false, + "error": null, + "completion_tokens": 24871, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/117", + "ok": true, + "error": null, + "completion_tokens": 734, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/118", + "ok": true, + "error": null, + "completion_tokens": 1459, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/119", + "ok": true, + "error": null, + "completion_tokens": 1023, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/120", + "ok": true, + "error": null, + "completion_tokens": 577, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/121", + "ok": true, + "error": null, + "completion_tokens": 341, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/122", + "ok": true, + "error": null, + "completion_tokens": 1064, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/123", + "ok": true, + "error": null, + "completion_tokens": 896, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/124", + "ok": true, + "error": null, + "completion_tokens": 4609, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/125", + "ok": true, + "error": null, + "completion_tokens": 1924, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/126", + "ok": true, + "error": null, + "completion_tokens": 1006, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/127", + "ok": true, + "error": null, + "completion_tokens": 5194, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/128", + "ok": true, + "error": null, + "completion_tokens": 428, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/129", + "ok": true, + "error": null, + "completion_tokens": 5066, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/130", + "ok": true, + "error": null, + "completion_tokens": 3302, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/131", + "ok": true, + "error": null, + "completion_tokens": 431, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/132", + "ok": true, + "error": null, + "completion_tokens": 11547, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/133", + "ok": true, + "error": null, + "completion_tokens": 401, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/134", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/135", + "ok": true, + "error": null, + "completion_tokens": 377, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/136", + "ok": true, + "error": null, + "completion_tokens": 407, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/137", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/138", + "ok": true, + "error": null, + "completion_tokens": 320, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/139", + "ok": true, + "error": null, + "completion_tokens": 515, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/140", + "ok": true, + "error": null, + "completion_tokens": 1141, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/141", + "ok": true, + "error": null, + "completion_tokens": 1500, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/142", + "ok": true, + "error": null, + "completion_tokens": 782, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/143", + "ok": true, + "error": null, + "completion_tokens": 565, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/144", + "ok": true, + "error": null, + "completion_tokens": 524, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/145", + "ok": false, + "error": null, + "completion_tokens": 30916, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/146", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/147", + "ok": false, + "error": null, + "completion_tokens": 1974, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/148", + "ok": true, + "error": null, + "completion_tokens": 564, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/149", + "ok": true, + "error": null, + "completion_tokens": 545, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/150", + "ok": true, + "error": null, + "completion_tokens": 360, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/151", + "ok": true, + "error": null, + "completion_tokens": 1517, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/152", + "ok": true, + "error": null, + "completion_tokens": 342, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/153", + "ok": true, + "error": null, + "completion_tokens": 917, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/154", + "ok": true, + "error": null, + "completion_tokens": 2078, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/155", + "ok": true, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/156", + "ok": true, + "error": null, + "completion_tokens": 639, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/157", + "ok": true, + "error": null, + "completion_tokens": 1207, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/158", + "ok": true, + "error": null, + "completion_tokens": 922, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/159", + "ok": true, + "error": null, + "completion_tokens": 466, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/160", + "ok": true, + "error": null, + "completion_tokens": 2360, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/161", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/162", + "ok": true, + "error": null, + "completion_tokens": 399, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/163", + "ok": true, + "error": null, + "completion_tokens": 1003, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "ifeval": { + "task": "ifeval", + "n": 100, + "hits": 89, + "accuracy_pct": 89.0, + "accuracy_ex_trunc_pct": 89.9, + "truncated": 1, + "completion_tokens_p50": 1230, + "completion_tokens_p90": 6372, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 578.0, + "gen_tokens": 267744, + "tok_per_s": 463.2, + "instruction_level_pct": 95.33, + "instructions_checked": 150, + "details": [ + { + "id": "ifeval-153", + "ok": true, + "error": null, + "completion_tokens": 2503, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-258", + "ok": true, + "error": null, + "completion_tokens": 380, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-368", + "ok": true, + "error": null, + "completion_tokens": 453, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-85", + "ok": true, + "error": null, + "completion_tokens": 1173, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-246", + "ok": true, + "error": null, + "completion_tokens": 1918, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-23", + "ok": true, + "error": null, + "completion_tokens": 5709, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-372", + "ok": true, + "error": null, + "completion_tokens": 2054, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-387", + "ok": true, + "error": null, + "completion_tokens": 907, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-138", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-56", + "ok": true, + "error": null, + "completion_tokens": 1247, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-277", + "ok": true, + "error": null, + "completion_tokens": 1886, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-407", + "ok": false, + "error": null, + "completion_tokens": 2267, + "reasoning_chars": 0, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-171", + "ok": true, + "error": null, + "completion_tokens": 1535, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-96", + "ok": true, + "error": null, + "completion_tokens": 2822, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-134", + "ok": false, + "error": null, + "completion_tokens": 11703, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-218", + "ok": true, + "error": null, + "completion_tokens": 1366, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-330", + "ok": false, + "error": null, + "completion_tokens": 2587, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-6", + "ok": true, + "error": null, + "completion_tokens": 1304, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-143", + "ok": true, + "error": null, + "completion_tokens": 348, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-121", + "ok": true, + "error": null, + "completion_tokens": 757, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-219", + "ok": true, + "error": null, + "completion_tokens": 788, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-524", + "ok": true, + "error": null, + "completion_tokens": 537, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-206", + "ok": true, + "error": null, + "completion_tokens": 743, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-144", + "ok": true, + "error": null, + "completion_tokens": 722, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-447", + "ok": true, + "error": null, + "completion_tokens": 314, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-492", + "ok": true, + "error": null, + "completion_tokens": 6372, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-485", + "ok": true, + "error": null, + "completion_tokens": 3723, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-220", + "ok": true, + "error": null, + "completion_tokens": 2083, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-412", + "ok": true, + "error": null, + "completion_tokens": 2718, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-342", + "ok": true, + "error": null, + "completion_tokens": 1230, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-222", + "ok": true, + "error": null, + "completion_tokens": 9120, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-131", + "ok": true, + "error": null, + "completion_tokens": 419, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-210", + "ok": true, + "error": null, + "completion_tokens": 349, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-137", + "ok": true, + "error": null, + "completion_tokens": 1243, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-464", + "ok": true, + "error": null, + "completion_tokens": 670, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-503", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-446", + "ok": true, + "error": null, + "completion_tokens": 769, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-2", + "ok": true, + "error": null, + "completion_tokens": 871, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-525", + "ok": true, + "error": null, + "completion_tokens": 503, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-27", + "ok": true, + "error": null, + "completion_tokens": 1839, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-136", + "ok": true, + "error": null, + "completion_tokens": 2355, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-286", + "ok": true, + "error": null, + "completion_tokens": 694, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-449", + "ok": true, + "error": null, + "completion_tokens": 716, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-55", + "ok": true, + "error": null, + "completion_tokens": 2413, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-93", + "ok": true, + "error": null, + "completion_tokens": 1199, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-399", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-19", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-229", + "ok": true, + "error": null, + "completion_tokens": 1502, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-181", + "ok": true, + "error": null, + "completion_tokens": 1121, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-362", + "ok": true, + "error": null, + "completion_tokens": 840, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-289", + "ok": true, + "error": null, + "completion_tokens": 1098, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-356", + "ok": true, + "error": null, + "completion_tokens": 566, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-115", + "ok": false, + "error": null, + "completion_tokens": 1377, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-404", + "ok": false, + "error": null, + "completion_tokens": 2509, + "reasoning_chars": 0, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-243", + "ok": false, + "error": null, + "completion_tokens": 26153, + "reasoning_chars": 0, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-90", + "ok": true, + "error": null, + "completion_tokens": 7850, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-54", + "ok": true, + "error": null, + "completion_tokens": 1367, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-276", + "ok": true, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-308", + "ok": true, + "error": null, + "completion_tokens": 2887, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-64", + "ok": true, + "error": null, + "completion_tokens": 1177, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-260", + "ok": true, + "error": null, + "completion_tokens": 481, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-337", + "ok": true, + "error": null, + "completion_tokens": 893, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-510", + "ok": true, + "error": null, + "completion_tokens": 3486, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-119", + "ok": true, + "error": null, + "completion_tokens": 2725, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-5", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-468", + "ok": true, + "error": null, + "completion_tokens": 7198, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-418", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-476", + "ok": true, + "error": null, + "completion_tokens": 3630, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-49", + "ok": true, + "error": null, + "completion_tokens": 6896, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-336", + "ok": true, + "error": null, + "completion_tokens": 355, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-422", + "ok": true, + "error": null, + "completion_tokens": 885, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-417", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-347", + "ok": true, + "error": null, + "completion_tokens": 13189, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-101", + "ok": true, + "error": null, + "completion_tokens": 986, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-209", + "ok": true, + "error": null, + "completion_tokens": 576, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-341", + "ok": true, + "error": null, + "completion_tokens": 2367, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-533", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-158", + "ok": true, + "error": null, + "completion_tokens": 621, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-321", + "ok": true, + "error": null, + "completion_tokens": 1416, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-311", + "ok": true, + "error": null, + "completion_tokens": 224, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-50", + "ok": false, + "error": null, + "completion_tokens": 10695, + "reasoning_chars": 0, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-518", + "ok": true, + "error": null, + "completion_tokens": 5696, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-374", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-317", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-512", + "ok": true, + "error": null, + "completion_tokens": 9785, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-102", + "ok": true, + "error": null, + "completion_tokens": 759, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-72", + "ok": false, + "error": null, + "completion_tokens": 1957, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-33", + "ok": false, + "error": null, + "completion_tokens": 1769, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-263", + "ok": true, + "error": null, + "completion_tokens": 1953, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-501", + "ok": true, + "error": null, + "completion_tokens": 3106, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-4", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-444", + "ok": true, + "error": null, + "completion_tokens": 1778, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-228", + "ok": true, + "error": null, + "completion_tokens": 1090, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-299", + "ok": true, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-278", + "ok": true, + "error": null, + "completion_tokens": 4160, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-154", + "ok": true, + "error": null, + "completion_tokens": 1083, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-301", + "ok": true, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-161", + "ok": true, + "error": null, + "completion_tokens": 5328, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-394", + "ok": false, + "error": null, + "completion_tokens": 2016, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-159", + "ok": true, + "error": null, + "completion_tokens": 498, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + } + ] + } + } + }, + "JonathanColetti-Qwen3.8-27B-Uncensored-w4a16-vision": { + "started": "2026-08-30 04:14:19", + "finished": "2026-08-30 06:10:59", + "seed": 20260816, + "thinking": true, + "tasks": { + "mmlu": { + "task": "mmlu", + "n": 150, + "hits": 126, + "accuracy_pct": 84.0, + "accuracy_ex_trunc_pct": 90.0, + "truncated": 10, + "completion_tokens_p50": 320, + "completion_tokens_p90": 9779, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 943.3, + "gen_tokens": 539547, + "tok_per_s": 572.0, + "details": [ + { + "id": "mmlu-12325", + "ok": true, + "error": null, + "completion_tokens": 839, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-154", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12844", + "ok": true, + "error": null, + "completion_tokens": 5792, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1153", + "ok": true, + "error": null, + "completion_tokens": 205, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11089", + "ok": true, + "error": null, + "completion_tokens": 1146, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12782", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-5422", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12363", + "ok": true, + "error": null, + "completion_tokens": 6558, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12556", + "ok": true, + "error": null, + "completion_tokens": 3417, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13163", + "ok": true, + "error": null, + "completion_tokens": 4408, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-270", + "ok": true, + "error": null, + "completion_tokens": 490, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6943", + "ok": true, + "error": null, + "completion_tokens": 570, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8330", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9276", + "ok": true, + "error": null, + "completion_tokens": 472, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-7584", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12819", + "ok": true, + "error": null, + "completion_tokens": 1571, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6163", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4099", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10389", + "ok": false, + "error": null, + "completion_tokens": 7356, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1539", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12013", + "ok": true, + "error": null, + "completion_tokens": 21095, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-7814", + "ok": true, + "error": null, + "completion_tokens": 247, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1106", + "ok": true, + "error": null, + "completion_tokens": 71, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1616", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-555", + "ok": true, + "error": null, + "completion_tokens": 162, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3698", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-618", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9842", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4229", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1022", + "ok": true, + "error": null, + "completion_tokens": 83, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6421", + "ok": true, + "error": null, + "completion_tokens": 2224, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1997", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8941", + "ok": false, + "error": null, + "completion_tokens": 15162, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-565", + "ok": true, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11364", + "ok": true, + "error": null, + "completion_tokens": 10319, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5959", + "ok": true, + "error": null, + "completion_tokens": 224, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13066", + "ok": false, + "error": null, + "completion_tokens": 4024, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3122", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3722", + "ok": true, + "error": null, + "completion_tokens": 362, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13839", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12570", + "ok": true, + "error": null, + "completion_tokens": 3918, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9959", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10791", + "ok": false, + "error": null, + "completion_tokens": 5621, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12268", + "ok": true, + "error": null, + "completion_tokens": 937, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9051", + "ok": true, + "error": null, + "completion_tokens": 371, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12887", + "ok": false, + "error": null, + "completion_tokens": 6809, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2667", + "ok": true, + "error": null, + "completion_tokens": 371, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3906", + "ok": true, + "error": null, + "completion_tokens": 578, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10304", + "ok": true, + "error": null, + "completion_tokens": 69, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12927", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6337", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3907", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12888", + "ok": false, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10629", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5801", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2936", + "ok": true, + "error": null, + "completion_tokens": 346, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-7389", + "ok": true, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9532", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-8705", + "ok": true, + "error": null, + "completion_tokens": 603, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11776", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-6836", + "ok": true, + "error": null, + "completion_tokens": 391, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-7856", + "ok": true, + "error": null, + "completion_tokens": 2179, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9697", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10825", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-11178", + "ok": true, + "error": null, + "completion_tokens": 9779, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11420", + "ok": true, + "error": null, + "completion_tokens": 4521, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4658", + "ok": true, + "error": null, + "completion_tokens": 276, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5403", + "ok": false, + "error": null, + "completion_tokens": 1270, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6160", + "ok": true, + "error": null, + "completion_tokens": 604, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6872", + "ok": false, + "error": null, + "completion_tokens": 10819, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9738", + "ok": true, + "error": null, + "completion_tokens": 293, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3076", + "ok": true, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13496", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11289", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-222", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12180", + "ok": true, + "error": null, + "completion_tokens": 801, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1143", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13086", + "ok": true, + "error": null, + "completion_tokens": 357, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-7038", + "ok": true, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13587", + "ok": false, + "error": null, + "completion_tokens": 659, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12349", + "ok": true, + "error": null, + "completion_tokens": 319, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11747", + "ok": true, + "error": null, + "completion_tokens": 731, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9848", + "ok": false, + "error": null, + "completion_tokens": 701, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3438", + "ok": true, + "error": null, + "completion_tokens": 303, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-89", + "ok": true, + "error": null, + "completion_tokens": 350, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4253", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5460", + "ok": true, + "error": null, + "completion_tokens": 790, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2494", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-11149", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-718", + "ok": true, + "error": null, + "completion_tokens": 520, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5330", + "ok": false, + "error": null, + "completion_tokens": 2435, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10716", + "ok": true, + "error": null, + "completion_tokens": 1619, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5636", + "ok": true, + "error": null, + "completion_tokens": 555, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11832", + "ok": true, + "error": null, + "completion_tokens": 647, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-95", + "ok": true, + "error": null, + "completion_tokens": 639, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13960", + "ok": true, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2072", + "ok": true, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12030", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-2931", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2441", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13666", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6575", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11625", + "ok": true, + "error": null, + "completion_tokens": 646, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13283", + "ok": true, + "error": null, + "completion_tokens": 8840, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4262", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12931", + "ok": true, + "error": null, + "completion_tokens": 321, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12150", + "ok": true, + "error": null, + "completion_tokens": 3265, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3678", + "ok": true, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4593", + "ok": true, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1563", + "ok": true, + "error": null, + "completion_tokens": 488, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13396", + "ok": true, + "error": null, + "completion_tokens": 320, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8049", + "ok": true, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8396", + "ok": true, + "error": null, + "completion_tokens": 1239, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-8493", + "ok": true, + "error": null, + "completion_tokens": 926, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13514", + "ok": true, + "error": null, + "completion_tokens": 604, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-3000", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1635", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1391", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-627", + "ok": false, + "error": null, + "completion_tokens": 13437, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1573", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6731", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11665", + "ok": true, + "error": null, + "completion_tokens": 2836, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2914", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6875", + "ok": true, + "error": null, + "completion_tokens": 2787, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2184", + "ok": true, + "error": null, + "completion_tokens": 237, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12801", + "ok": true, + "error": null, + "completion_tokens": 258, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-6717", + "ok": true, + "error": null, + "completion_tokens": 290, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12464", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2787", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11549", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-842", + "ok": true, + "error": null, + "completion_tokens": 298, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13868", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9647", + "ok": true, + "error": null, + "completion_tokens": 203, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9502", + "ok": true, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12441", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-4392", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-5036", + "ok": true, + "error": null, + "completion_tokens": 5105, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12606", + "ok": false, + "error": null, + "completion_tokens": 4328, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-1400", + "ok": true, + "error": null, + "completion_tokens": 383, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13979", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-13634", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-11226", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlu-625", + "ok": true, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-257", + "ok": true, + "error": null, + "completion_tokens": 587, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-12853", + "ok": true, + "error": null, + "completion_tokens": 246, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-451", + "ok": true, + "error": null, + "completion_tokens": 625, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-2074", + "ok": true, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-9117", + "ok": false, + "error": null, + "completion_tokens": 5712, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlu-10748", + "ok": true, + "error": null, + "completion_tokens": 543, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "cmmlu": { + "task": "cmmlu", + "n": 150, + "hits": 134, + "accuracy_pct": 89.33, + "accuracy_ex_trunc_pct": 89.93, + "truncated": 1, + "completion_tokens_p50": 353, + "completion_tokens_p90": 4401, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 713.6, + "gen_tokens": 306615, + "tok_per_s": 429.7, + "details": [ + { + "id": "cmmlu-chinese_teacher_qualification-63", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-54", + "ok": true, + "error": null, + "completion_tokens": 1473, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_science-91", + "ok": true, + "error": null, + "completion_tokens": 303, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-260", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-32", + "ok": true, + "error": null, + "completion_tokens": 275, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-29", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-200", + "ok": false, + "error": null, + "completion_tokens": 13518, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-98", + "ok": true, + "error": null, + "completion_tokens": 258, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-44", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-118", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-116", + "ok": false, + "error": null, + "completion_tokens": 10952, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-43", + "ok": true, + "error": null, + "completion_tokens": 380, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-145", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-247", + "ok": true, + "error": null, + "completion_tokens": 257, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sociology-224", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-26", + "ok": true, + "error": null, + "completion_tokens": 725, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-105", + "ok": true, + "error": null, + "completion_tokens": 3074, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-logical-15", + "ok": true, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-62", + "ok": true, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-nutrition-74", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-181", + "ok": true, + "error": null, + "completion_tokens": 1216, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-127", + "ok": true, + "error": null, + "completion_tokens": 671, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_geography-107", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-agronomy-18", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-82", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_food_culture-38", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-198", + "ok": true, + "error": null, + "completion_tokens": 574, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-96", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-73", + "ok": true, + "error": null, + "completion_tokens": 28302, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-genetics-20", + "ok": true, + "error": null, + "completion_tokens": 316, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-logical-95", + "ok": true, + "error": null, + "completion_tokens": 261, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-genetics-88", + "ok": true, + "error": null, + "completion_tokens": 1335, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-192", + "ok": true, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-66", + "ok": true, + "error": null, + "completion_tokens": 397, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-100", + "ok": true, + "error": null, + "completion_tokens": 411, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-136", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-management-16", + "ok": false, + "error": null, + "completion_tokens": 551, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-logical-90", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sports_science-41", + "ok": false, + "error": null, + "completion_tokens": 28472, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-345", + "ok": true, + "error": null, + "completion_tokens": 4401, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sociology-103", + "ok": false, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-agronomy-93", + "ok": true, + "error": null, + "completion_tokens": 499, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-88", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-management-206", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-105", + "ok": true, + "error": null, + "completion_tokens": 487, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-economics-90", + "ok": true, + "error": null, + "completion_tokens": 90, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-190", + "ok": false, + "error": null, + "completion_tokens": 422, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-international_law-39", + "ok": true, + "error": null, + "completion_tokens": 191, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-155", + "ok": true, + "error": null, + "completion_tokens": 2145, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-24", + "ok": true, + "error": null, + "completion_tokens": 1222, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-6", + "ok": true, + "error": null, + "completion_tokens": 669, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-31", + "ok": true, + "error": null, + "completion_tokens": 797, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-243", + "ok": true, + "error": null, + "completion_tokens": 833, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-63", + "ok": true, + "error": null, + "completion_tokens": 5072, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sports_science-135", + "ok": true, + "error": null, + "completion_tokens": 1108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_science-89", + "ok": true, + "error": null, + "completion_tokens": 80, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-91", + "ok": true, + "error": null, + "completion_tokens": 10725, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-120", + "ok": true, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-genetics-62", + "ok": true, + "error": null, + "completion_tokens": 1400, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-management-106", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-food_science-106", + "ok": true, + "error": null, + "completion_tokens": 20133, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-24", + "ok": false, + "error": null, + "completion_tokens": 20887, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-157", + "ok": true, + "error": null, + "completion_tokens": 1835, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marketing-41", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-115", + "ok": true, + "error": null, + "completion_tokens": 2459, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-journalism-29", + "ok": false, + "error": null, + "completion_tokens": 1658, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-67", + "ok": true, + "error": null, + "completion_tokens": 272, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-161", + "ok": true, + "error": null, + "completion_tokens": 462, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-virology-120", + "ok": true, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-education-117", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-169", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-129", + "ok": true, + "error": null, + "completion_tokens": 601, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-19", + "ok": true, + "error": null, + "completion_tokens": 2290, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-8", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marketing-72", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-public_relations-139", + "ok": false, + "error": null, + "completion_tokens": 510, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_security-22", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-7", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_law-108", + "ok": true, + "error": null, + "completion_tokens": 534, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-31", + "ok": true, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-131", + "ok": true, + "error": null, + "completion_tokens": 634, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_science-35", + "ok": false, + "error": null, + "completion_tokens": 7553, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_actuarial_science-38", + "ok": true, + "error": null, + "completion_tokens": 2446, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marketing-165", + "ok": true, + "error": null, + "completion_tokens": 90, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marketing-101", + "ok": true, + "error": null, + "completion_tokens": 564, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-107", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-8", + "ok": true, + "error": null, + "completion_tokens": 14850, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_medical_statistics-86", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-security_study-30", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-118", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_law-24", + "ok": false, + "error": null, + "completion_tokens": 3821, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-46", + "ok": true, + "error": null, + "completion_tokens": 312, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-anatomy-21", + "ok": false, + "error": null, + "completion_tokens": 10047, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-145", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-arts-91", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-109", + "ok": false, + "error": null, + "completion_tokens": 172, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-44", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-74", + "ok": true, + "error": null, + "completion_tokens": 172, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-63", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-astronomy-146", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-109", + "ok": true, + "error": null, + "completion_tokens": 298, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-construction_project_management-16", + "ok": true, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-113", + "ok": false, + "error": null, + "completion_tokens": 583, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-102", + "ok": true, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-207", + "ok": true, + "error": null, + "completion_tokens": 411, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-economics-19", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-management-164", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-338", + "ok": true, + "error": null, + "completion_tokens": 586, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-7", + "ok": true, + "error": null, + "completion_tokens": 510, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-virology-92", + "ok": true, + "error": null, + "completion_tokens": 741, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-47", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-astronomy-140", + "ok": true, + "error": null, + "completion_tokens": 145, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-100", + "ok": true, + "error": null, + "completion_tokens": 659, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-37", + "ok": true, + "error": null, + "completion_tokens": 1106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-191", + "ok": true, + "error": null, + "completion_tokens": 1457, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-34", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-88", + "ok": true, + "error": null, + "completion_tokens": 374, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-public_relations-77", + "ok": false, + "error": null, + "completion_tokens": 377, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-306", + "ok": true, + "error": null, + "completion_tokens": 2758, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-40", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_chemistry-7", + "ok": true, + "error": null, + "completion_tokens": 4046, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-152", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_science-1", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-94", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-109", + "ok": true, + "error": null, + "completion_tokens": 1115, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-164", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sports_science-89", + "ok": true, + "error": null, + "completion_tokens": 3271, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-economics-17", + "ok": true, + "error": null, + "completion_tokens": 1818, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-arts-2", + "ok": true, + "error": null, + "completion_tokens": 71, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-genetics-51", + "ok": true, + "error": null, + "completion_tokens": 558, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-agronomy-141", + "ok": true, + "error": null, + "completion_tokens": 329, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-computer_science-196", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-genetics-97", + "ok": true, + "error": null, + "completion_tokens": 450, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-202", + "ok": true, + "error": null, + "completion_tokens": 5839, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-management-195", + "ok": true, + "error": null, + "completion_tokens": 1369, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-21", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-sociology-49", + "ok": true, + "error": null, + "completion_tokens": 938, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-94", + "ok": true, + "error": null, + "completion_tokens": 301, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-8", + "ok": true, + "error": null, + "completion_tokens": 6604, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-98", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-28", + "ok": true, + "error": null, + "completion_tokens": 377, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-74", + "ok": true, + "error": null, + "completion_tokens": 5080, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-121", + "ok": true, + "error": null, + "completion_tokens": 360, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-120", + "ok": true, + "error": null, + "completion_tokens": 402, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-105", + "ok": true, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-35", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "cmmlu-professional_accounting-51", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-49", + "ok": true, + "error": null, + "completion_tokens": 2804, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-130", + "ok": true, + "error": null, + "completion_tokens": 379, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "cmmlu-virology-140", + "ok": true, + "error": null, + "completion_tokens": 331, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "mmlu_pro": { + "task": "mmlu_pro", + "n": 150, + "hits": 125, + "accuracy_pct": 83.33, + "accuracy_ex_trunc_pct": 88.03, + "truncated": 8, + "completion_tokens_p50": 506, + "completion_tokens_p90": 7652, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 949.4, + "gen_tokens": 520513, + "tok_per_s": 548.2, + "details": [ + { + "id": "mmlupro-5831", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2299", + "ok": true, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9280", + "ok": true, + "error": null, + "completion_tokens": 7652, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5486", + "ok": true, + "error": null, + "completion_tokens": 2295, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5761", + "ok": true, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6589", + "ok": true, + "error": null, + "completion_tokens": 612, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5389", + "ok": true, + "error": null, + "completion_tokens": 319, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5131", + "ok": true, + "error": null, + "completion_tokens": 500, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1677", + "ok": true, + "error": null, + "completion_tokens": 4127, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4388", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-3285", + "ok": true, + "error": null, + "completion_tokens": 1164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4118", + "ok": false, + "error": null, + "completion_tokens": 1789, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5344", + "ok": true, + "error": null, + "completion_tokens": 378, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11696", + "ok": true, + "error": null, + "completion_tokens": 1911, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-858", + "ok": true, + "error": null, + "completion_tokens": 436, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-214", + "ok": true, + "error": null, + "completion_tokens": 2818, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6609", + "ok": true, + "error": null, + "completion_tokens": 498, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2083", + "ok": true, + "error": null, + "completion_tokens": 450, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9752", + "ok": false, + "error": null, + "completion_tokens": 5071, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7213", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5234", + "ok": true, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8570", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5612", + "ok": false, + "error": null, + "completion_tokens": 413, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2071", + "ok": true, + "error": null, + "completion_tokens": 379, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9842", + "ok": true, + "error": null, + "completion_tokens": 409, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4665", + "ok": true, + "error": null, + "completion_tokens": 1358, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5564", + "ok": true, + "error": null, + "completion_tokens": 4471, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6475", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2153", + "ok": true, + "error": null, + "completion_tokens": 506, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5812", + "ok": true, + "error": null, + "completion_tokens": 65, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3407", + "ok": true, + "error": null, + "completion_tokens": 8082, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2879", + "ok": false, + "error": null, + "completion_tokens": 2649, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8266", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1916", + "ok": false, + "error": null, + "completion_tokens": 3274, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3486", + "ok": true, + "error": null, + "completion_tokens": 2247, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11275", + "ok": true, + "error": null, + "completion_tokens": 512, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6040", + "ok": true, + "error": null, + "completion_tokens": 606, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8565", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6848", + "ok": true, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8926", + "ok": true, + "error": null, + "completion_tokens": 208, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8458", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7854", + "ok": true, + "error": null, + "completion_tokens": 454, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3962", + "ok": true, + "error": null, + "completion_tokens": 2098, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6470", + "ok": true, + "error": null, + "completion_tokens": 1082, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3813", + "ok": true, + "error": null, + "completion_tokens": 363, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8026", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11030", + "ok": true, + "error": null, + "completion_tokens": 1136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8151", + "ok": true, + "error": null, + "completion_tokens": 88, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5286", + "ok": true, + "error": null, + "completion_tokens": 404, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3888", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11676", + "ok": true, + "error": null, + "completion_tokens": 1343, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8762", + "ok": false, + "error": null, + "completion_tokens": 1396, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-12160", + "ok": true, + "error": null, + "completion_tokens": 684, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4022", + "ok": true, + "error": null, + "completion_tokens": 3181, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10092", + "ok": true, + "error": null, + "completion_tokens": 439, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7364", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6001", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10873", + "ok": true, + "error": null, + "completion_tokens": 374, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9777", + "ok": true, + "error": null, + "completion_tokens": 6641, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8723", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4595", + "ok": true, + "error": null, + "completion_tokens": 628, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8423", + "ok": true, + "error": null, + "completion_tokens": 1530, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11051", + "ok": false, + "error": null, + "completion_tokens": 385, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-836", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11297", + "ok": true, + "error": null, + "completion_tokens": 17604, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3950", + "ok": true, + "error": null, + "completion_tokens": 378, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1562", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-2041", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3473", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11253", + "ok": true, + "error": null, + "completion_tokens": 4236, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3858", + "ok": true, + "error": null, + "completion_tokens": 695, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2231", + "ok": true, + "error": null, + "completion_tokens": 1608, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2190", + "ok": true, + "error": null, + "completion_tokens": 523, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6930", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6558", + "ok": true, + "error": null, + "completion_tokens": 448, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5371", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-1766", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-6045", + "ok": true, + "error": null, + "completion_tokens": 721, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2298", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6400", + "ok": true, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6499", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7936", + "ok": true, + "error": null, + "completion_tokens": 398, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8566", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11633", + "ok": true, + "error": null, + "completion_tokens": 605, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9843", + "ok": false, + "error": null, + "completion_tokens": 6844, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11758", + "ok": false, + "error": null, + "completion_tokens": 17427, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2843", + "ok": true, + "error": null, + "completion_tokens": 2015, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7483", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11379", + "ok": true, + "error": null, + "completion_tokens": 1507, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4866", + "ok": true, + "error": null, + "completion_tokens": 843, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-818", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2785", + "ok": true, + "error": null, + "completion_tokens": 2550, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8119", + "ok": true, + "error": null, + "completion_tokens": 998, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2214", + "ok": true, + "error": null, + "completion_tokens": 10659, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5774", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-6049", + "ok": false, + "error": null, + "completion_tokens": 1540, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7219", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1660", + "ok": false, + "error": null, + "completion_tokens": 833, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4767", + "ok": false, + "error": null, + "completion_tokens": 18775, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5964", + "ok": false, + "error": null, + "completion_tokens": 1018, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2743", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4062", + "ok": true, + "error": null, + "completion_tokens": 834, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7368", + "ok": true, + "error": null, + "completion_tokens": 444, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-827", + "ok": false, + "error": null, + "completion_tokens": 6588, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11877", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-7557", + "ok": true, + "error": null, + "completion_tokens": 3639, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9998", + "ok": true, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10098", + "ok": false, + "error": null, + "completion_tokens": 7639, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1744", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-9210", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11248", + "ok": true, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11187", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2799", + "ok": true, + "error": null, + "completion_tokens": 212, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4479", + "ok": true, + "error": null, + "completion_tokens": 2314, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10223", + "ok": true, + "error": null, + "completion_tokens": 721, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5698", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8291", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3516", + "ok": true, + "error": null, + "completion_tokens": 378, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5466", + "ok": true, + "error": null, + "completion_tokens": 847, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4851", + "ok": false, + "error": null, + "completion_tokens": 1255, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-733", + "ok": true, + "error": null, + "completion_tokens": 2197, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-6995", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2573", + "ok": true, + "error": null, + "completion_tokens": 1263, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8481", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-5938", + "ok": true, + "error": null, + "completion_tokens": 1101, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10708", + "ok": true, + "error": null, + "completion_tokens": 332, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8476", + "ok": true, + "error": null, + "completion_tokens": 401, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-7990", + "ok": true, + "error": null, + "completion_tokens": 414, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2409", + "ok": true, + "error": null, + "completion_tokens": 517, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11551", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-674", + "ok": true, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-1978", + "ok": true, + "error": null, + "completion_tokens": 9348, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10420", + "ok": true, + "error": null, + "completion_tokens": 258, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3445", + "ok": true, + "error": null, + "completion_tokens": 6059, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9488", + "ok": true, + "error": null, + "completion_tokens": 409, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4993", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4316", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4730", + "ok": true, + "error": null, + "completion_tokens": 302, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11570", + "ok": true, + "error": null, + "completion_tokens": 419, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-2149", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8322", + "ok": true, + "error": null, + "completion_tokens": 3008, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-10993", + "ok": true, + "error": null, + "completion_tokens": 5485, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4588", + "ok": true, + "error": null, + "completion_tokens": 5395, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-3258", + "ok": true, + "error": null, + "completion_tokens": 763, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-8629", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-637", + "ok": true, + "error": null, + "completion_tokens": 445, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-4257", + "ok": true, + "error": null, + "completion_tokens": 545, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-9719", + "ok": true, + "error": null, + "completion_tokens": 2803, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "mmlupro-11714", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "mmlupro-9678", + "ok": false, + "error": null, + "completion_tokens": 14426, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "ceval": { + "task": "ceval", + "n": 150, + "hits": 127, + "accuracy_pct": 84.67, + "accuracy_ex_trunc_pct": 86.39, + "truncated": 3, + "completion_tokens_p50": 432, + "completion_tokens_p90": 6069, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 720.6, + "gen_tokens": 352420, + "tok_per_s": 489.1, + "details": [ + { + "id": "ceval-high_school_chinese-1", + "ok": false, + "error": null, + "completion_tokens": 14466, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-10", + "ok": true, + "error": null, + "completion_tokens": 1006, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-tax_accountant-41", + "ok": false, + "error": null, + "completion_tokens": 2007, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_physics-3", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-21", + "ok": false, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-4", + "ok": true, + "error": null, + "completion_tokens": 909, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_history-21", + "ok": true, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-2", + "ok": true, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_programming-6", + "ok": true, + "error": null, + "completion_tokens": 421, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-12", + "ok": false, + "error": null, + "completion_tokens": 5092, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-19", + "ok": true, + "error": null, + "completion_tokens": 405, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_chemistry-9", + "ok": true, + "error": null, + "completion_tokens": 6069, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-plant_protection-15", + "ok": true, + "error": null, + "completion_tokens": 452, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-22", + "ok": false, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-plant_protection-16", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-business_administration-11", + "ok": true, + "error": null, + "completion_tokens": 626, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-fire_engineer-18", + "ok": true, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-24", + "ok": true, + "error": null, + "completion_tokens": 777, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-29", + "ok": true, + "error": null, + "completion_tokens": 1909, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-2", + "ok": true, + "error": null, + "completion_tokens": 1226, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-11", + "ok": false, + "error": null, + "completion_tokens": 1128, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-18", + "ok": true, + "error": null, + "completion_tokens": 1007, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-0", + "ok": false, + "error": null, + "completion_tokens": 6077, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-14", + "ok": true, + "error": null, + "completion_tokens": 378, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-business_administration-27", + "ok": true, + "error": null, + "completion_tokens": 2766, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-art_studies-19", + "ok": true, + "error": null, + "completion_tokens": 1605, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-21", + "ok": true, + "error": null, + "completion_tokens": 6087, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-tax_accountant-33", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-20", + "ok": true, + "error": null, + "completion_tokens": 558, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-legal_professional-10", + "ok": true, + "error": null, + "completion_tokens": 807, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_programming-19", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-computer_network-7", + "ok": true, + "error": null, + "completion_tokens": 98, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-41", + "ok": true, + "error": null, + "completion_tokens": 868, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-44", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-9", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-28", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-39", + "ok": true, + "error": null, + "completion_tokens": 441, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-1", + "ok": false, + "error": null, + "completion_tokens": 1458, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-business_administration-24", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-legal_professional-5", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "ceval-computer_architecture-15", + "ok": true, + "error": null, + "completion_tokens": 208, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-39", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-law-23", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-7", + "ok": true, + "error": null, + "completion_tokens": 1053, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-17", + "ok": true, + "error": null, + "completion_tokens": 1521, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_chemistry-6", + "ok": true, + "error": null, + "completion_tokens": 403, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-15", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-4", + "ok": true, + "error": null, + "completion_tokens": 15738, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-5", + "ok": true, + "error": null, + "completion_tokens": 2108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-50", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_politics-11", + "ok": true, + "error": null, + "completion_tokens": 806, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-tax_accountant-32", + "ok": true, + "error": null, + "completion_tokens": 4857, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-16", + "ok": true, + "error": null, + "completion_tokens": 628, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-fire_engineer-0", + "ok": true, + "error": null, + "completion_tokens": 475, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-sports_science-9", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-art_studies-15", + "ok": true, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-2", + "ok": false, + "error": null, + "completion_tokens": 3980, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-fire_engineer-7", + "ok": false, + "error": null, + "completion_tokens": 11116, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-plant_protection-11", + "ok": true, + "error": null, + "completion_tokens": 1904, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-operating_system-18", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-4", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_programming-17", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_programming-1", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-business_administration-32", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-33", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-education_science-25", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-40", + "ok": true, + "error": null, + "completion_tokens": 294, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-legal_professional-7", + "ok": true, + "error": null, + "completion_tokens": 9491, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-5", + "ok": true, + "error": null, + "completion_tokens": 415, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-fire_engineer-12", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-4", + "ok": true, + "error": null, + "completion_tokens": 88, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-38", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "ceval-electrical_engineer-22", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-0", + "ok": true, + "error": null, + "completion_tokens": 1201, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-15", + "ok": false, + "error": null, + "completion_tokens": 3937, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-45", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-6", + "ok": true, + "error": null, + "completion_tokens": 475, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-legal_professional-20", + "ok": true, + "error": null, + "completion_tokens": 4652, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-basic_medicine-2", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-17", + "ok": true, + "error": null, + "completion_tokens": 1292, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-16", + "ok": false, + "error": null, + "completion_tokens": 1101, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-7", + "ok": true, + "error": null, + "completion_tokens": 375, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_biology-16", + "ok": false, + "error": null, + "completion_tokens": 428, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-10", + "ok": true, + "error": null, + "completion_tokens": 401, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-32", + "ok": true, + "error": null, + "completion_tokens": 3685, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-0", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-12", + "ok": true, + "error": null, + "completion_tokens": 1430, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-logic-14", + "ok": true, + "error": null, + "completion_tokens": 1193, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-26", + "ok": true, + "error": null, + "completion_tokens": 3491, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-3", + "ok": true, + "error": null, + "completion_tokens": 443, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-3", + "ok": true, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-computer_network-13", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-sports_science-5", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-27", + "ok": false, + "error": null, + "completion_tokens": 5486, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-fire_engineer-21", + "ok": true, + "error": null, + "completion_tokens": 16138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-10", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-16", + "ok": true, + "error": null, + "completion_tokens": 4807, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-49", + "ok": true, + "error": null, + "completion_tokens": 3132, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-13", + "ok": false, + "error": null, + "completion_tokens": 1261, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-business_administration-5", + "ok": true, + "error": null, + "completion_tokens": 16082, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-2", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-27", + "ok": false, + "error": null, + "completion_tokens": 4671, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_biology-8", + "ok": true, + "error": null, + "completion_tokens": 400, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_physics-14", + "ok": true, + "error": null, + "completion_tokens": 768, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_history-14", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-31", + "ok": true, + "error": null, + "completion_tokens": 674, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-basic_medicine-13", + "ok": true, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-13", + "ok": true, + "error": null, + "completion_tokens": 7156, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_programming-12", + "ok": true, + "error": null, + "completion_tokens": 480, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-civil_servant-44", + "ok": false, + "error": null, + "completion_tokens": 9065, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-3", + "ok": true, + "error": null, + "completion_tokens": 2896, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-13", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "ceval-middle_school_mathematics-6", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_politics-19", + "ok": true, + "error": null, + "completion_tokens": 463, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-environmental_impact_assessment_engineer-27", + "ok": false, + "error": null, + "completion_tokens": 2775, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-law-10", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-law-0", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-3", + "ok": true, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-6", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_chinese-13", + "ok": true, + "error": null, + "completion_tokens": 890, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-43", + "ok": false, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-42", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-13", + "ok": true, + "error": null, + "completion_tokens": 495, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-discrete_mathematics-1", + "ok": true, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-11", + "ok": true, + "error": null, + "completion_tokens": 1160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 137, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-plant_protection-2", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-1", + "ok": true, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_physics-16", + "ok": true, + "error": null, + "completion_tokens": 8457, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-11", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-5", + "ok": true, + "error": null, + "completion_tokens": 172, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-40", + "ok": true, + "error": null, + "completion_tokens": 289, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-6", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-middle_school_mathematics-10", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-10", + "ok": true, + "error": null, + "completion_tokens": 7655, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-education_science-17", + "ok": true, + "error": null, + "completion_tokens": 3788, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-tax_accountant-11", + "ok": true, + "error": null, + "completion_tokens": 875, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-16", + "ok": true, + "error": null, + "completion_tokens": 72, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-education_science-1", + "ok": false, + "error": null, + "completion_tokens": 2176, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_physics-0", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-37", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-9", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-high_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 617, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-5", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-18", + "ok": true, + "error": null, + "completion_tokens": 669, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-physician-30", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-accountant-21", + "ok": true, + "error": null, + "completion_tokens": 362, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_economics-42", + "ok": true, + "error": null, + "completion_tokens": 588, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-5", + "ok": true, + "error": null, + "completion_tokens": 951, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "ceval-college_chemistry-4", + "ok": true, + "error": null, + "completion_tokens": 294, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "arc": { + "task": "arc", + "n": 150, + "hits": 144, + "accuracy_pct": 96.0, + "accuracy_ex_trunc_pct": 96.0, + "truncated": 0, + "completion_tokens_p50": 154, + "completion_tokens_p90": 352, + "completion_tokens_max": 5934, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 98.3, + "gen_tokens": 37652, + "tok_per_s": 382.9, + "details": [ + { + "id": "arc-Mercury_415084", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7249708", + "ok": true, + "error": null, + "completion_tokens": 266, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_4_24", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7056823", + "ok": true, + "error": null, + "completion_tokens": 294, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415534", + "ok": true, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405827", + "ok": false, + "error": null, + "completion_tokens": 5934, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7263183", + "ok": true, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7106960", + "ok": true, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_7", + "ok": true, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7182245", + "ok": true, + "error": null, + "completion_tokens": 1258, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416177", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-ACTAAP_2011_5_8", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-TAKS_2009_8_36", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_401699", + "ok": true, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7107240", + "ok": true, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_4_26", + "ok": true, + "error": null, + "completion_tokens": 111, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402638", + "ok": true, + "error": null, + "completion_tokens": 1911, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7026723", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415422", + "ok": true, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-LEAP_2000_8_2", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2006_9_4", + "ok": true, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401272", + "ok": true, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416169", + "ok": true, + "error": null, + "completion_tokens": 65, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7116235", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7109463", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7032428", + "ok": true, + "error": null, + "completion_tokens": 734, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-AIMS_2008_8_11", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_410334", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_36", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415719", + "ok": true, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_17", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402256", + "ok": true, + "error": null, + "completion_tokens": 217, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_400443", + "ok": true, + "error": null, + "completion_tokens": 86, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7133245", + "ok": true, + "error": null, + "completion_tokens": 289, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413458", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401372", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401116", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-LEAP__5_10312", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-LEAP_2007_4_10280", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_416683", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7190120", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401295", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_4_28", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7223493", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7018410", + "ok": true, + "error": null, + "completion_tokens": 230, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_1998_8_24", + "ok": true, + "error": null, + "completion_tokens": 894, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_31", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405501", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408030", + "ok": true, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_401684", + "ok": true, + "error": null, + "completion_tokens": 162, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7041668", + "ok": true, + "error": null, + "completion_tokens": 145, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7205870", + "ok": true, + "error": null, + "completion_tokens": 45, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7210140", + "ok": true, + "error": null, + "completion_tokens": 341, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_405873", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400052", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7018270", + "ok": true, + "error": null, + "completion_tokens": 885, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_406550", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7150728", + "ok": true, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400839", + "ok": true, + "error": null, + "completion_tokens": 131, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_30", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400383", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7236740", + "ok": true, + "error": null, + "completion_tokens": 208, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-CSZ_2005_5_CSZ10021", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7093958", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7026355", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408324", + "ok": true, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-LEAP__7_10348", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402629", + "ok": true, + "error": null, + "completion_tokens": 56, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400046", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7180810", + "ok": true, + "error": null, + "completion_tokens": 434, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_180005", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-AIMS_2009_8_14", + "ok": true, + "error": null, + "completion_tokens": 332, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MDSA_2007_5_16", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_LBS10662", + "ok": true, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_23", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2012_8_19", + "ok": false, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7041650", + "ok": true, + "error": null, + "completion_tokens": 137, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416181", + "ok": true, + "error": null, + "completion_tokens": 269, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-TIMSS_1995_8_J6", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2014_5_13", + "ok": true, + "error": null, + "completion_tokens": 319, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7106750", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_1999_4_23", + "ok": true, + "error": null, + "completion_tokens": 250, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MEA_2013_5_12", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_8_42", + "ok": false, + "error": null, + "completion_tokens": 409, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408367", + "ok": true, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7250058", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_414041", + "ok": true, + "error": null, + "completion_tokens": 264, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401290", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_409295", + "ok": true, + "error": null, + "completion_tokens": 346, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7211680", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_37", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413439", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_32", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7216423", + "ok": true, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-ACTAAP_2015_7_9", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7230405", + "ok": true, + "error": null, + "completion_tokens": 549, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7025008", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_416536", + "ok": true, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-TIMSS_2007_8_pg60", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MDSA_2009_4_34", + "ok": true, + "error": null, + "completion_tokens": 379, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_406833", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_409647", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_184363", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7107170", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416104", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_10-v1", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7202843", + "ok": true, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7029803", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7164623", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_26", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7263060", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7268328", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402282", + "ok": true, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7029558", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_4_21", + "ok": true, + "error": null, + "completion_tokens": 69, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7186358", + "ok": true, + "error": null, + "completion_tokens": 172, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-LEAP_2011_8_10434", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2012_8_23651", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_407125", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7268258", + "ok": true, + "error": null, + "completion_tokens": 689, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7027405", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2004_8_7", + "ok": true, + "error": null, + "completion_tokens": 598, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_191625", + "ok": true, + "error": null, + "completion_tokens": 385, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_6", + "ok": true, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7219678", + "ok": true, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401678", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_401312", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_8_42", + "ok": false, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7030468", + "ok": true, + "error": null, + "completion_tokens": 631, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_175385", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_175560", + "ok": true, + "error": null, + "completion_tokens": 83, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_5", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_409009", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7128923", + "ok": false, + "error": null, + "completion_tokens": 383, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_8_9", + "ok": false, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7042945", + "ok": true, + "error": null, + "completion_tokens": 269, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_8_47", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7185343", + "ok": true, + "error": null, + "completion_tokens": 294, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7218400", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7221988", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_405950", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7222530", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7102323", + "ok": true, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MDSA_2009_5_39", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-MCAS_1999_8_5", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400661", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_7081603", + "ok": true, + "error": null, + "completion_tokens": 314, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-VASoL_2008_3_25", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413628", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "arc-Mercury_406427", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "truthfulqa": { + "task": "truthfulqa", + "n": 150, + "hits": 120, + "accuracy_pct": 80.0, + "accuracy_ex_trunc_pct": 82.19, + "truncated": 4, + "completion_tokens_p50": 513, + "completion_tokens_p90": 5735, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 686.3, + "gen_tokens": 391895, + "tok_per_s": 571.0, + "details": [ + { + "id": "tqa-210", + "ok": true, + "error": null, + "completion_tokens": 3202, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-76", + "ok": true, + "error": null, + "completion_tokens": 5628, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-461", + "ok": true, + "error": null, + "completion_tokens": 627, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-218", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-254", + "ok": true, + "error": null, + "completion_tokens": 5735, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-635", + "ok": true, + "error": null, + "completion_tokens": 245, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-193", + "ok": true, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-148", + "ok": true, + "error": null, + "completion_tokens": 289, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-400", + "ok": false, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-766", + "ok": true, + "error": null, + "completion_tokens": 504, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-240", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "tqa-50", + "ok": true, + "error": null, + "completion_tokens": 517, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-83", + "ok": true, + "error": null, + "completion_tokens": 463, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-381", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-325", + "ok": true, + "error": null, + "completion_tokens": 9150, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-655", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-721", + "ok": true, + "error": null, + "completion_tokens": 11417, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-391", + "ok": true, + "error": null, + "completion_tokens": 1547, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-401", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "tqa-625", + "ok": true, + "error": null, + "completion_tokens": 301, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-80", + "ok": false, + "error": null, + "completion_tokens": 1512, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-439", + "ok": true, + "error": null, + "completion_tokens": 310, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-66", + "ok": true, + "error": null, + "completion_tokens": 1788, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-155", + "ok": true, + "error": null, + "completion_tokens": 618, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-516", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-522", + "ok": true, + "error": null, + "completion_tokens": 3144, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-47", + "ok": false, + "error": null, + "completion_tokens": 3871, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-343", + "ok": true, + "error": null, + "completion_tokens": 551, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-126", + "ok": true, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-77", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-613", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-590", + "ok": true, + "error": null, + "completion_tokens": 265, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-674", + "ok": true, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-438", + "ok": true, + "error": null, + "completion_tokens": 4147, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-503", + "ok": true, + "error": null, + "completion_tokens": 2357, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-639", + "ok": true, + "error": null, + "completion_tokens": 303, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-111", + "ok": true, + "error": null, + "completion_tokens": 4087, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-289", + "ok": true, + "error": null, + "completion_tokens": 1941, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-705", + "ok": false, + "error": null, + "completion_tokens": 15000, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-373", + "ok": true, + "error": null, + "completion_tokens": 829, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-363", + "ok": true, + "error": null, + "completion_tokens": 3897, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-792", + "ok": true, + "error": null, + "completion_tokens": 562, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-113", + "ok": false, + "error": null, + "completion_tokens": 4090, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-74", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-634", + "ok": true, + "error": null, + "completion_tokens": 1938, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-804", + "ok": true, + "error": null, + "completion_tokens": 2267, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-626", + "ok": true, + "error": null, + "completion_tokens": 247, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-676", + "ok": true, + "error": null, + "completion_tokens": 342, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-174", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-455", + "ok": false, + "error": null, + "completion_tokens": 5089, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-725", + "ok": true, + "error": null, + "completion_tokens": 345, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-296", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-535", + "ok": true, + "error": null, + "completion_tokens": 4021, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-172", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-23", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-43", + "ok": true, + "error": null, + "completion_tokens": 192, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-38", + "ok": true, + "error": null, + "completion_tokens": 392, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-97", + "ok": false, + "error": null, + "completion_tokens": 798, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-657", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-814", + "ok": true, + "error": null, + "completion_tokens": 423, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-346", + "ok": true, + "error": null, + "completion_tokens": 416, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-365", + "ok": true, + "error": null, + "completion_tokens": 226, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-384", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-677", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "tqa-486", + "ok": true, + "error": null, + "completion_tokens": 363, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-55", + "ok": true, + "error": null, + "completion_tokens": 518, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-680", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-510", + "ok": false, + "error": null, + "completion_tokens": 1619, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-128", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "tqa-805", + "ok": false, + "error": null, + "completion_tokens": 1877, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-394", + "ok": false, + "error": null, + "completion_tokens": 1626, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-435", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-285", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-687", + "ok": false, + "error": null, + "completion_tokens": 1136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-588", + "ok": true, + "error": null, + "completion_tokens": 1402, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-100", + "ok": true, + "error": null, + "completion_tokens": 940, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-399", + "ok": true, + "error": null, + "completion_tokens": 563, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-266", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-693", + "ok": true, + "error": null, + "completion_tokens": 633, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-532", + "ok": false, + "error": null, + "completion_tokens": 13593, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-558", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-319", + "ok": true, + "error": null, + "completion_tokens": 1907, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-459", + "ok": false, + "error": null, + "completion_tokens": 4274, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-675", + "ok": true, + "error": null, + "completion_tokens": 377, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-129", + "ok": true, + "error": null, + "completion_tokens": 1742, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-331", + "ok": false, + "error": null, + "completion_tokens": 2109, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-12", + "ok": false, + "error": null, + "completion_tokens": 8492, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-164", + "ok": true, + "error": null, + "completion_tokens": 5900, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-306", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-275", + "ok": true, + "error": null, + "completion_tokens": 301, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-418", + "ok": true, + "error": null, + "completion_tokens": 821, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-264", + "ok": true, + "error": null, + "completion_tokens": 547, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-94", + "ok": false, + "error": null, + "completion_tokens": 1379, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-389", + "ok": true, + "error": null, + "completion_tokens": 506, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-427", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-269", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-643", + "ok": false, + "error": null, + "completion_tokens": 3042, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-7", + "ok": false, + "error": null, + "completion_tokens": 2960, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-419", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-616", + "ok": true, + "error": null, + "completion_tokens": 166, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-699", + "ok": true, + "error": null, + "completion_tokens": 162, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-473", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-523", + "ok": true, + "error": null, + "completion_tokens": 855, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-214", + "ok": true, + "error": null, + "completion_tokens": 473, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-376", + "ok": true, + "error": null, + "completion_tokens": 309, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-123", + "ok": true, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-200", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-493", + "ok": true, + "error": null, + "completion_tokens": 729, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-474", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-387", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-746", + "ok": true, + "error": null, + "completion_tokens": 131, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-790", + "ok": true, + "error": null, + "completion_tokens": 557, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-181", + "ok": true, + "error": null, + "completion_tokens": 2626, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-413", + "ok": true, + "error": null, + "completion_tokens": 513, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-349", + "ok": false, + "error": null, + "completion_tokens": 1774, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-434", + "ok": true, + "error": null, + "completion_tokens": 719, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-601", + "ok": true, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-489", + "ok": true, + "error": null, + "completion_tokens": 116, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-286", + "ok": false, + "error": null, + "completion_tokens": 332, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-488", + "ok": true, + "error": null, + "completion_tokens": 1359, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-759", + "ok": true, + "error": null, + "completion_tokens": 3434, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-273", + "ok": true, + "error": null, + "completion_tokens": 1156, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-59", + "ok": true, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-315", + "ok": true, + "error": null, + "completion_tokens": 337, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-425", + "ok": true, + "error": null, + "completion_tokens": 728, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-58", + "ok": true, + "error": null, + "completion_tokens": 725, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-691", + "ok": true, + "error": null, + "completion_tokens": 202, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-497", + "ok": true, + "error": null, + "completion_tokens": 265, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-515", + "ok": true, + "error": null, + "completion_tokens": 203, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-398", + "ok": true, + "error": null, + "completion_tokens": 2324, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-670", + "ok": true, + "error": null, + "completion_tokens": 12449, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-170", + "ok": false, + "error": null, + "completion_tokens": 522, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-334", + "ok": false, + "error": null, + "completion_tokens": 8635, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-307", + "ok": false, + "error": null, + "completion_tokens": 4483, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-495", + "ok": false, + "error": null, + "completion_tokens": 13321, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-597", + "ok": true, + "error": null, + "completion_tokens": 4124, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-775", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-186", + "ok": true, + "error": null, + "completion_tokens": 478, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-733", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-308", + "ok": true, + "error": null, + "completion_tokens": 7061, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-5", + "ok": true, + "error": null, + "completion_tokens": 318, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-231", + "ok": false, + "error": null, + "completion_tokens": 11172, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-356", + "ok": true, + "error": null, + "completion_tokens": 882, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-810", + "ok": true, + "error": null, + "completion_tokens": 296, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-270", + "ok": true, + "error": null, + "completion_tokens": 3975, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-13", + "ok": false, + "error": null, + "completion_tokens": 414, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-114", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-4", + "ok": true, + "error": null, + "completion_tokens": 458, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-417", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "tqa-750", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "gsm8k": { + "task": "gsm8k", + "n": 100, + "hits": 99, + "accuracy_pct": 99.0, + "accuracy_ex_trunc_pct": 99.0, + "truncated": 0, + "completion_tokens_p50": 238, + "completion_tokens_p90": 464, + "completion_tokens_max": 13222, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 207.9, + "gen_tokens": 45961, + "tok_per_s": 221.1, + "details": [ + { + "id": "gsm8k-144", + "ok": true, + "error": null, + "completion_tokens": 337, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-627", + "ok": true, + "error": null, + "completion_tokens": 371, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-195", + "ok": true, + "error": null, + "completion_tokens": 364, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-201", + "ok": true, + "error": null, + "completion_tokens": 2982, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-730", + "ok": true, + "error": null, + "completion_tokens": 145, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-166", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1141", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-832", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-53", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1042", + "ok": false, + "error": null, + "completion_tokens": 3590, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-609", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-318", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-892", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-204", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1120", + "ok": true, + "error": null, + "completion_tokens": 274, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1040", + "ok": true, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-556", + "ok": true, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1015", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-206", + "ok": true, + "error": null, + "completion_tokens": 375, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1058", + "ok": true, + "error": null, + "completion_tokens": 338, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1044", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-131", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1154", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-158", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1149", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1156", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-60", + "ok": true, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-918", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1010", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1032", + "ok": true, + "error": null, + "completion_tokens": 303, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1053", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-577", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-935", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-692", + "ok": true, + "error": null, + "completion_tokens": 391, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-153", + "ok": true, + "error": null, + "completion_tokens": 546, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-40", + "ok": true, + "error": null, + "completion_tokens": 1003, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-613", + "ok": true, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-875", + "ok": true, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-189", + "ok": true, + "error": null, + "completion_tokens": 875, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-800", + "ok": true, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-169", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1169", + "ok": true, + "error": null, + "completion_tokens": 464, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-211", + "ok": true, + "error": null, + "completion_tokens": 217, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1183", + "ok": true, + "error": null, + "completion_tokens": 721, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-193", + "ok": true, + "error": null, + "completion_tokens": 268, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-230", + "ok": true, + "error": null, + "completion_tokens": 370, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-787", + "ok": true, + "error": null, + "completion_tokens": 200, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-83", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1318", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-733", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1205", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-88", + "ok": true, + "error": null, + "completion_tokens": 172, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-509", + "ok": true, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-311", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-874", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-287", + "ok": true, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1055", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1246", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-975", + "ok": true, + "error": null, + "completion_tokens": 273, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-280", + "ok": true, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-541", + "ok": true, + "error": null, + "completion_tokens": 375, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-974", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-827", + "ok": true, + "error": null, + "completion_tokens": 366, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-125", + "ok": true, + "error": null, + "completion_tokens": 351, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-453", + "ok": true, + "error": null, + "completion_tokens": 212, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-942", + "ok": true, + "error": null, + "completion_tokens": 265, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1199", + "ok": true, + "error": null, + "completion_tokens": 325, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-884", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-855", + "ok": true, + "error": null, + "completion_tokens": 442, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1270", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-349", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1231", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1106", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-803", + "ok": true, + "error": null, + "completion_tokens": 187, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-922", + "ok": true, + "error": null, + "completion_tokens": 632, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-152", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-920", + "ok": true, + "error": null, + "completion_tokens": 259, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-207", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-297", + "ok": true, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-786", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-101", + "ok": true, + "error": null, + "completion_tokens": 259, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1301", + "ok": true, + "error": null, + "completion_tokens": 356, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-27", + "ok": true, + "error": null, + "completion_tokens": 166, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1005", + "ok": true, + "error": null, + "completion_tokens": 808, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-659", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-340", + "ok": true, + "error": null, + "completion_tokens": 13222, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-288", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-763", + "ok": true, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-840", + "ok": true, + "error": null, + "completion_tokens": 277, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-742", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-222", + "ok": true, + "error": null, + "completion_tokens": 162, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-468", + "ok": true, + "error": null, + "completion_tokens": 391, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-220", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-662", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-269", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-1193", + "ok": true, + "error": null, + "completion_tokens": 404, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-270", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-114", + "ok": true, + "error": null, + "completion_tokens": 480, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-728", + "ok": true, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "gsm8k-396", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "math500": { + "task": "math500", + "n": 100, + "hits": 77, + "accuracy_pct": 77.0, + "accuracy_ex_trunc_pct": 77.78, + "truncated": 1, + "completion_tokens_p50": 458, + "completion_tokens_p90": 4398, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 566.6, + "gen_tokens": 184600, + "tok_per_s": 325.8, + "details": [ + { + "id": "math-test/algebra/1553.json", + "ok": true, + "error": null, + "completion_tokens": 461, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/538.json", + "ok": true, + "error": null, + "completion_tokens": 617, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/691.json", + "ok": true, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/1090.json", + "ok": true, + "error": null, + "completion_tokens": 728, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/116.json", + "ok": true, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1255.json", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1143.json", + "ok": true, + "error": null, + "completion_tokens": 706, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/precalculus/1201.json", + "ok": true, + "error": null, + "completion_tokens": 4398, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/964.json", + "ok": true, + "error": null, + "completion_tokens": 5756, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/662.json", + "ok": false, + "error": null, + "completion_tokens": 12268, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/153.json", + "ok": false, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/276.json", + "ok": true, + "error": null, + "completion_tokens": 499, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1840.json", + "ok": true, + "error": null, + "completion_tokens": 522, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1072.json", + "ok": false, + "error": null, + "completion_tokens": 537, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1907.json", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/567.json", + "ok": true, + "error": null, + "completion_tokens": 513, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1922.json", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1779.json", + "ok": true, + "error": null, + "completion_tokens": 5444, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/951.json", + "ok": true, + "error": null, + "completion_tokens": 273, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1197.json", + "ok": false, + "error": null, + "completion_tokens": 6942, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/853.json", + "ok": true, + "error": null, + "completion_tokens": 1223, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/516.json", + "ok": true, + "error": null, + "completion_tokens": 733, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1114.json", + "ok": false, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/1065.json", + "ok": true, + "error": null, + "completion_tokens": 742, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1232.json", + "ok": false, + "error": null, + "completion_tokens": 499, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/769.json", + "ok": true, + "error": null, + "completion_tokens": 915, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1791.json", + "ok": false, + "error": null, + "completion_tokens": 1384, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/246.json", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1166.json", + "ok": true, + "error": null, + "completion_tokens": 2936, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1410.json", + "ok": false, + "error": null, + "completion_tokens": 412, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1686.json", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/754.json", + "ok": false, + "error": null, + "completion_tokens": 1129, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/477.json", + "ok": true, + "error": null, + "completion_tokens": 8420, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2680.json", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/23957.json", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1865.json", + "ok": true, + "error": null, + "completion_tokens": 2247, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1014.json", + "ok": true, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/431.json", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1425.json", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2584.json", + "ok": false, + "error": null, + "completion_tokens": 321, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2470.json", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/547.json", + "ok": true, + "error": null, + "completion_tokens": 3035, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/precalculus/1105.json", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/347.json", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/1257.json", + "ok": true, + "error": null, + "completion_tokens": 400, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1126.json", + "ok": true, + "error": null, + "completion_tokens": 634, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/239.json", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1961.json", + "ok": false, + "error": null, + "completion_tokens": 360, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1388.json", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1078.json", + "ok": true, + "error": null, + "completion_tokens": 724, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2157.json", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/339.json", + "ok": true, + "error": null, + "completion_tokens": 3175, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1761.json", + "ok": false, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1960.json", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/2086.json", + "ok": true, + "error": null, + "completion_tokens": 538, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1462.json", + "ok": true, + "error": null, + "completion_tokens": 2923, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1060.json", + "ok": true, + "error": null, + "completion_tokens": 191, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/1055.json", + "ok": true, + "error": null, + "completion_tokens": 2726, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2517.json", + "ok": false, + "error": null, + "completion_tokens": 615, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/996.json", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/893.json", + "ok": true, + "error": null, + "completion_tokens": 458, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/51.json", + "ok": false, + "error": null, + "completion_tokens": 715, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/precalculus/1291.json", + "ok": true, + "error": null, + "completion_tokens": 504, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/694.json", + "ok": true, + "error": null, + "completion_tokens": 425, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/711.json", + "ok": true, + "error": null, + "completion_tokens": 24878, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1655.json", + "ok": false, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1936.json", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1572.json", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/509.json", + "ok": false, + "error": null, + "completion_tokens": 781, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/864.json", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1405.json", + "ok": true, + "error": null, + "completion_tokens": 3537, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/192.json", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2102.json", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/843.json", + "ok": true, + "error": null, + "completion_tokens": 415, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/187.json", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/612.json", + "ok": true, + "error": null, + "completion_tokens": 680, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/615.json", + "ok": true, + "error": null, + "completion_tokens": 5281, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1908.json", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/928.json", + "ok": true, + "error": null, + "completion_tokens": 1009, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/505.json", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1930.json", + "ok": false, + "error": null, + "completion_tokens": 8571, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1934.json", + "ok": true, + "error": null, + "completion_tokens": 351, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1139.json", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "math-test/intermediate_algebra/1000.json", + "ok": true, + "error": null, + "completion_tokens": 1781, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/183.json", + "ok": true, + "error": null, + "completion_tokens": 2416, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/1275.json", + "ok": true, + "error": null, + "completion_tokens": 452, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2700.json", + "ok": false, + "error": null, + "completion_tokens": 1586, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2257.json", + "ok": false, + "error": null, + "completion_tokens": 372, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/14.json", + "ok": false, + "error": null, + "completion_tokens": 1893, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/precalculus/1300.json", + "ok": false, + "error": null, + "completion_tokens": 850, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/number_theory/515.json", + "ok": true, + "error": null, + "completion_tokens": 355, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1423.json", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1102.json", + "ok": true, + "error": null, + "completion_tokens": 1118, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1646.json", + "ok": true, + "error": null, + "completion_tokens": 5804, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/1272.json", + "ok": false, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/25149.json", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/precalculus/697.json", + "ok": true, + "error": null, + "completion_tokens": 2878, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/algebra/2036.json", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/prealgebra/2037.json", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "math-test/geometry/353.json", + "ok": true, + "error": null, + "completion_tokens": 397, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "bbh": { + "task": "bbh", + "n": 150, + "hits": 138, + "accuracy_pct": 92.0, + "accuracy_ex_trunc_pct": 95.17, + "truncated": 5, + "completion_tokens_p50": 517, + "completion_tokens_p90": 5084, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 801.4, + "gen_tokens": 413241, + "tok_per_s": 515.6, + "details": [ + { + "id": "bbh-sports_understanding-101", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-60", + "ok": true, + "error": null, + "completion_tokens": 524, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-113", + "ok": true, + "error": null, + "completion_tokens": 2149, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-web_of_lies-31", + "ok": true, + "error": null, + "completion_tokens": 1213, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-144", + "ok": true, + "error": null, + "completion_tokens": 30974, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-28", + "ok": true, + "error": null, + "completion_tokens": 307, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-116", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-67", + "ok": true, + "error": null, + "completion_tokens": 278, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-3", + "ok": true, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-29", + "ok": true, + "error": null, + "completion_tokens": 237, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-192", + "ok": true, + "error": null, + "completion_tokens": 616, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-196", + "ok": true, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-10", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-83", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-76", + "ok": true, + "error": null, + "completion_tokens": 776, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-212", + "ok": true, + "error": null, + "completion_tokens": 511, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-87", + "ok": true, + "error": null, + "completion_tokens": 779, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-1", + "ok": true, + "error": null, + "completion_tokens": 1333, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-140", + "ok": true, + "error": null, + "completion_tokens": 3866, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-8", + "ok": true, + "error": null, + "completion_tokens": 415, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-138", + "ok": true, + "error": null, + "completion_tokens": 619, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-221", + "ok": true, + "error": null, + "completion_tokens": 928, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-causal_judgement-136", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "bbh-penguins_in_a_table-14", + "ok": true, + "error": null, + "completion_tokens": 83, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-0", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-207", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-32", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-83", + "ok": true, + "error": null, + "completion_tokens": 587, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-220", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-234", + "ok": true, + "error": null, + "completion_tokens": 870, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-ruin_names-61", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-21", + "ok": true, + "error": null, + "completion_tokens": 579, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-31", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-13", + "ok": true, + "error": null, + "completion_tokens": 616, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-211", + "ok": true, + "error": null, + "completion_tokens": 804, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-37", + "ok": true, + "error": null, + "completion_tokens": 514, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-225", + "ok": true, + "error": null, + "completion_tokens": 487, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-90", + "ok": true, + "error": null, + "completion_tokens": 325, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-101", + "ok": true, + "error": null, + "completion_tokens": 3119, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-web_of_lies-179", + "ok": true, + "error": null, + "completion_tokens": 597, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-173", + "ok": true, + "error": null, + "completion_tokens": 303, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-105", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-88", + "ok": true, + "error": null, + "completion_tokens": 718, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-68", + "ok": true, + "error": null, + "completion_tokens": 2573, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-185", + "ok": true, + "error": null, + "completion_tokens": 21529, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-227", + "ok": true, + "error": null, + "completion_tokens": 203, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-129", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-238", + "ok": true, + "error": null, + "completion_tokens": 224, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-202", + "ok": true, + "error": null, + "completion_tokens": 326, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-93", + "ok": true, + "error": null, + "completion_tokens": 356, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-93", + "ok": false, + "error": null, + "completion_tokens": 6454, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-161", + "ok": true, + "error": null, + "completion_tokens": 212, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-sports_understanding-46", + "ok": true, + "error": null, + "completion_tokens": 9991, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-227", + "ok": true, + "error": null, + "completion_tokens": 1129, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-158", + "ok": true, + "error": null, + "completion_tokens": 316, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-205", + "ok": true, + "error": null, + "completion_tokens": 6220, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-89", + "ok": true, + "error": null, + "completion_tokens": 442, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-177", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-65", + "ok": true, + "error": null, + "completion_tokens": 262, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-72", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-49", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-54", + "ok": true, + "error": null, + "completion_tokens": 2270, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-64", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-86", + "ok": true, + "error": null, + "completion_tokens": 1845, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-108", + "ok": true, + "error": null, + "completion_tokens": 111, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-194", + "ok": true, + "error": null, + "completion_tokens": 409, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-194", + "ok": true, + "error": null, + "completion_tokens": 284, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-dyck_languages-75", + "ok": true, + "error": null, + "completion_tokens": 10091, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-106", + "ok": true, + "error": null, + "completion_tokens": 502, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-81", + "ok": true, + "error": null, + "completion_tokens": 633, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-242", + "ok": true, + "error": null, + "completion_tokens": 186, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-navigate-150", + "ok": true, + "error": null, + "completion_tokens": 431, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-causal_judgement-25", + "ok": true, + "error": null, + "completion_tokens": 5084, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-139", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-12", + "ok": true, + "error": null, + "completion_tokens": 449, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-178", + "ok": true, + "error": null, + "completion_tokens": 456, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-160", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-dyck_languages-115", + "ok": true, + "error": null, + "completion_tokens": 1366, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-44", + "ok": true, + "error": null, + "completion_tokens": 1327, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-189", + "ok": true, + "error": null, + "completion_tokens": 200, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-102", + "ok": true, + "error": null, + "completion_tokens": 1242, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-38", + "ok": true, + "error": null, + "completion_tokens": 1097, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-141", + "ok": true, + "error": null, + "completion_tokens": 298, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-38", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-168", + "ok": true, + "error": null, + "completion_tokens": 564, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-dyck_languages-212", + "ok": true, + "error": null, + "completion_tokens": 20764, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-navigate-245", + "ok": true, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-222", + "ok": true, + "error": null, + "completion_tokens": 1306, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-244", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-35", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-126", + "ok": false, + "error": null, + "completion_tokens": 918, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-90", + "ok": false, + "error": null, + "completion_tokens": 1023, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-5", + "ok": true, + "error": null, + "completion_tokens": 514, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-sports_understanding-241", + "ok": true, + "error": null, + "completion_tokens": 563, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-234", + "ok": true, + "error": null, + "completion_tokens": 1032, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-233", + "ok": true, + "error": null, + "completion_tokens": 467, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-5", + "ok": true, + "error": null, + "completion_tokens": 1459, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-causal_judgement-152", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "bbh-logical_deduction_three_objects-9", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-209", + "ok": true, + "error": null, + "completion_tokens": 2399, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-171", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-navigate-160", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-166", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "bbh-formal_fallacies-213", + "ok": true, + "error": null, + "completion_tokens": 904, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-240", + "ok": true, + "error": null, + "completion_tokens": 3727, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-95", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-227", + "ok": true, + "error": null, + "completion_tokens": 607, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-snarks-96", + "ok": true, + "error": null, + "completion_tokens": 707, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-33", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-4", + "ok": true, + "error": null, + "completion_tokens": 571, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-180", + "ok": true, + "error": null, + "completion_tokens": 3316, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-74", + "ok": true, + "error": null, + "completion_tokens": 379, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-88", + "ok": true, + "error": null, + "completion_tokens": 321, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-10", + "ok": true, + "error": null, + "completion_tokens": 517, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-dyck_languages-191", + "ok": false, + "error": null, + "completion_tokens": 11387, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-71", + "ok": true, + "error": null, + "completion_tokens": 1365, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-150", + "ok": true, + "error": null, + "completion_tokens": 518, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-188", + "ok": true, + "error": null, + "completion_tokens": 1235, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-191", + "ok": true, + "error": null, + "completion_tokens": 451, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-web_of_lies-203", + "ok": true, + "error": null, + "completion_tokens": 1277, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-ruin_names-237", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-24", + "ok": true, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-27", + "ok": true, + "error": null, + "completion_tokens": 567, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-177", + "ok": true, + "error": null, + "completion_tokens": 2027, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-192", + "ok": true, + "error": null, + "completion_tokens": 119, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-242", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-dyck_languages-51", + "ok": false, + "error": null, + "completion_tokens": 13105, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-96", + "ok": true, + "error": null, + "completion_tokens": 911, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-word_sorting-100", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-204", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-58", + "ok": true, + "error": null, + "completion_tokens": 727, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-object_counting-157", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-104", + "ok": true, + "error": null, + "completion_tokens": 431, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-47", + "ok": true, + "error": null, + "completion_tokens": 402, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-date_understanding-150", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-23", + "ok": true, + "error": null, + "completion_tokens": 542, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-sports_understanding-68", + "ok": true, + "error": null, + "completion_tokens": 980, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-174", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-108", + "ok": true, + "error": null, + "completion_tokens": 1343, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-203", + "ok": false, + "error": null, + "completion_tokens": 2246, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-115", + "ok": true, + "error": null, + "completion_tokens": 421, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-hyperbaton-223", + "ok": true, + "error": null, + "completion_tokens": 438, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-221", + "ok": true, + "error": null, + "completion_tokens": 14029, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-5", + "ok": true, + "error": null, + "completion_tokens": 542, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-233", + "ok": true, + "error": null, + "completion_tokens": 1323, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-ruin_names-3", + "ok": true, + "error": null, + "completion_tokens": 605, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-119", + "ok": true, + "error": null, + "completion_tokens": 1858, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-197", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "bbh-snarks-103", + "ok": true, + "error": null, + "completion_tokens": 2321, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "bbh-causal_judgement-178", + "ok": false, + "error": null, + "completion_tokens": 4119, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "humaneval": { + "task": "humaneval", + "n": 164, + "hits": 137, + "accuracy_pct": 83.54, + "accuracy_ex_trunc_pct": 88.96, + "truncated": 10, + "completion_tokens_p50": 758, + "completion_tokens_p90": 11639, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 1006.1, + "gen_tokens": 617430, + "tok_per_s": 613.7, + "details": [ + { + "id": "HumanEval/0", + "ok": true, + "error": null, + "completion_tokens": 944, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/1", + "ok": false, + "error": null, + "completion_tokens": 2736, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/2", + "ok": true, + "error": null, + "completion_tokens": 736, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/3", + "ok": true, + "error": null, + "completion_tokens": 815, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/4", + "ok": true, + "error": null, + "completion_tokens": 1225, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/5", + "ok": false, + "error": null, + "completion_tokens": 588, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/6", + "ok": true, + "error": null, + "completion_tokens": 827, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/7", + "ok": false, + "error": null, + "completion_tokens": 423, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/8", + "ok": true, + "error": null, + "completion_tokens": 549, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/9", + "ok": false, + "error": null, + "completion_tokens": 485, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/10", + "ok": true, + "error": null, + "completion_tokens": 13837, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/11", + "ok": true, + "error": null, + "completion_tokens": 1578, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/12", + "ok": false, + "error": null, + "completion_tokens": 754, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/13", + "ok": true, + "error": null, + "completion_tokens": 461, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/14", + "ok": true, + "error": null, + "completion_tokens": 1909, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/15", + "ok": true, + "error": null, + "completion_tokens": 228, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/16", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/17", + "ok": false, + "error": null, + "completion_tokens": 740, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/18", + "ok": true, + "error": null, + "completion_tokens": 3133, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/19", + "ok": true, + "error": null, + "completion_tokens": 781, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/20", + "ok": false, + "error": null, + "completion_tokens": 1363, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/21", + "ok": false, + "error": null, + "completion_tokens": 830, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/22", + "ok": false, + "error": null, + "completion_tokens": 3780, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/23", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/24", + "ok": true, + "error": null, + "completion_tokens": 3456, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/25", + "ok": true, + "error": null, + "completion_tokens": 5281, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/26", + "ok": false, + "error": null, + "completion_tokens": 782, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/27", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/28", + "ok": false, + "error": null, + "completion_tokens": 487, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/29", + "ok": false, + "error": null, + "completion_tokens": 662, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/30", + "ok": true, + "error": null, + "completion_tokens": 259, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/31", + "ok": true, + "error": null, + "completion_tokens": 1047, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/32", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/33", + "ok": true, + "error": null, + "completion_tokens": 591, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/34", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/35", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/36", + "ok": true, + "error": null, + "completion_tokens": 1240, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/37", + "ok": true, + "error": null, + "completion_tokens": 594, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/38", + "ok": false, + "error": null, + "completion_tokens": 747, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/39", + "ok": true, + "error": null, + "completion_tokens": 5417, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/40", + "ok": true, + "error": null, + "completion_tokens": 5025, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/41", + "ok": true, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/42", + "ok": true, + "error": null, + "completion_tokens": 355, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/43", + "ok": true, + "error": null, + "completion_tokens": 497, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/44", + "ok": true, + "error": null, + "completion_tokens": 1333, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/45", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/46", + "ok": true, + "error": null, + "completion_tokens": 1753, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/47", + "ok": false, + "error": null, + "completion_tokens": 19219, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/48", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/49", + "ok": true, + "error": null, + "completion_tokens": 523, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/50", + "ok": false, + "error": null, + "completion_tokens": 284, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/51", + "ok": true, + "error": null, + "completion_tokens": 326, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/52", + "ok": true, + "error": null, + "completion_tokens": 367, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/53", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/54", + "ok": true, + "error": null, + "completion_tokens": 439, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/55", + "ok": true, + "error": null, + "completion_tokens": 377, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/56", + "ok": true, + "error": null, + "completion_tokens": 490, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/57", + "ok": true, + "error": null, + "completion_tokens": 439, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/58", + "ok": true, + "error": null, + "completion_tokens": 403, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/59", + "ok": true, + "error": null, + "completion_tokens": 863, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/60", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/61", + "ok": true, + "error": null, + "completion_tokens": 406, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/62", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/63", + "ok": true, + "error": null, + "completion_tokens": 1065, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/64", + "ok": true, + "error": null, + "completion_tokens": 689, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/65", + "ok": true, + "error": null, + "completion_tokens": 1556, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/66", + "ok": true, + "error": null, + "completion_tokens": 508, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/67", + "ok": true, + "error": null, + "completion_tokens": 558, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/68", + "ok": true, + "error": null, + "completion_tokens": 886, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/69", + "ok": true, + "error": null, + "completion_tokens": 647, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/70", + "ok": true, + "error": null, + "completion_tokens": 610, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/71", + "ok": true, + "error": null, + "completion_tokens": 1948, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/72", + "ok": true, + "error": null, + "completion_tokens": 391, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/73", + "ok": true, + "error": null, + "completion_tokens": 603, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/74", + "ok": true, + "error": null, + "completion_tokens": 568, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/75", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/76", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/77", + "ok": true, + "error": null, + "completion_tokens": 2212, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/78", + "ok": true, + "error": null, + "completion_tokens": 527, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/79", + "ok": true, + "error": null, + "completion_tokens": 541, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/80", + "ok": true, + "error": null, + "completion_tokens": 426, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/81", + "ok": true, + "error": null, + "completion_tokens": 1368, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/82", + "ok": true, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/83", + "ok": true, + "error": null, + "completion_tokens": 1107, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/84", + "ok": true, + "error": null, + "completion_tokens": 403, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/85", + "ok": true, + "error": null, + "completion_tokens": 442, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/86", + "ok": true, + "error": null, + "completion_tokens": 1238, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/87", + "ok": true, + "error": null, + "completion_tokens": 747, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/88", + "ok": true, + "error": null, + "completion_tokens": 742, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/89", + "ok": true, + "error": null, + "completion_tokens": 924, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/90", + "ok": true, + "error": null, + "completion_tokens": 816, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/91", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/92", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/93", + "ok": true, + "error": null, + "completion_tokens": 3117, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/94", + "ok": true, + "error": null, + "completion_tokens": 14504, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/95", + "ok": true, + "error": null, + "completion_tokens": 1177, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/96", + "ok": true, + "error": null, + "completion_tokens": 840, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/97", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/98", + "ok": true, + "error": null, + "completion_tokens": 526, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/99", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/100", + "ok": true, + "error": null, + "completion_tokens": 722, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/101", + "ok": true, + "error": null, + "completion_tokens": 456, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/102", + "ok": true, + "error": null, + "completion_tokens": 5762, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/103", + "ok": true, + "error": null, + "completion_tokens": 2262, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/104", + "ok": true, + "error": null, + "completion_tokens": 513, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/105", + "ok": true, + "error": null, + "completion_tokens": 1176, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/106", + "ok": true, + "error": null, + "completion_tokens": 1171, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/107", + "ok": true, + "error": null, + "completion_tokens": 607, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/108", + "ok": true, + "error": null, + "completion_tokens": 1211, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/109", + "ok": true, + "error": null, + "completion_tokens": 2413, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/110", + "ok": true, + "error": null, + "completion_tokens": 982, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/111", + "ok": true, + "error": null, + "completion_tokens": 703, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/112", + "ok": true, + "error": null, + "completion_tokens": 519, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/113", + "ok": true, + "error": null, + "completion_tokens": 1733, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/114", + "ok": true, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/115", + "ok": true, + "error": null, + "completion_tokens": 975, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/116", + "ok": false, + "error": null, + "completion_tokens": 22109, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/117", + "ok": true, + "error": null, + "completion_tokens": 684, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/118", + "ok": true, + "error": null, + "completion_tokens": 2237, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/119", + "ok": true, + "error": null, + "completion_tokens": 794, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/120", + "ok": true, + "error": null, + "completion_tokens": 560, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/121", + "ok": true, + "error": null, + "completion_tokens": 424, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/122", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/123", + "ok": true, + "error": null, + "completion_tokens": 915, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/124", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/125", + "ok": true, + "error": null, + "completion_tokens": 3053, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/126", + "ok": true, + "error": null, + "completion_tokens": 908, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/127", + "ok": true, + "error": null, + "completion_tokens": 898, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/128", + "ok": true, + "error": null, + "completion_tokens": 849, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/129", + "ok": true, + "error": null, + "completion_tokens": 11639, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/130", + "ok": true, + "error": null, + "completion_tokens": 2569, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/131", + "ok": true, + "error": null, + "completion_tokens": 502, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/132", + "ok": true, + "error": null, + "completion_tokens": 13778, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/133", + "ok": true, + "error": null, + "completion_tokens": 1032, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/134", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/135", + "ok": true, + "error": null, + "completion_tokens": 661, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/136", + "ok": true, + "error": null, + "completion_tokens": 597, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/137", + "ok": true, + "error": null, + "completion_tokens": 2440, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/138", + "ok": true, + "error": null, + "completion_tokens": 451, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/139", + "ok": true, + "error": null, + "completion_tokens": 462, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/140", + "ok": true, + "error": null, + "completion_tokens": 762, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/141", + "ok": true, + "error": null, + "completion_tokens": 1337, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/142", + "ok": true, + "error": null, + "completion_tokens": 656, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/143", + "ok": true, + "error": null, + "completion_tokens": 659, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/144", + "ok": true, + "error": null, + "completion_tokens": 570, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/145", + "ok": false, + "error": null, + "completion_tokens": 32463, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/146", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 0, + "truncated": true + }, + { + "id": "HumanEval/147", + "ok": true, + "error": null, + "completion_tokens": 2809, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/148", + "ok": true, + "error": null, + "completion_tokens": 758, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/149", + "ok": true, + "error": null, + "completion_tokens": 965, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/150", + "ok": true, + "error": null, + "completion_tokens": 371, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/151", + "ok": true, + "error": null, + "completion_tokens": 6479, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/152", + "ok": true, + "error": null, + "completion_tokens": 356, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/153", + "ok": true, + "error": null, + "completion_tokens": 1417, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/154", + "ok": true, + "error": null, + "completion_tokens": 736, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/155", + "ok": true, + "error": null, + "completion_tokens": 487, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/156", + "ok": true, + "error": null, + "completion_tokens": 561, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/157", + "ok": true, + "error": null, + "completion_tokens": 4971, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/158", + "ok": true, + "error": null, + "completion_tokens": 1031, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/159", + "ok": true, + "error": null, + "completion_tokens": 466, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/160", + "ok": true, + "error": null, + "completion_tokens": 2065, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/161", + "ok": true, + "error": null, + "completion_tokens": 524, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/162", + "ok": true, + "error": null, + "completion_tokens": 464, + "reasoning_chars": 0, + "truncated": false + }, + { + "id": "HumanEval/163", + "ok": true, + "error": null, + "completion_tokens": 2284, + "reasoning_chars": 0, + "truncated": false + } + ] + }, + "ifeval": { + "task": "ifeval", + "n": 100, + "hits": 93, + "accuracy_pct": 93.0, + "accuracy_ex_trunc_pct": 93.0, + "truncated": 0, + "completion_tokens_p50": 1053, + "completion_tokens_p90": 4032, + "completion_tokens_max": 16872, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "errors": 0, + "elapsed_s": 302.9, + "gen_tokens": 183475, + "tok_per_s": 605.8, + "instruction_level_pct": 98.0, + "instructions_checked": 150, + "details": [ + { + "id": "ifeval-153", + "ok": true, + "error": null, + "completion_tokens": 2013, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-258", + "ok": true, + "error": null, + "completion_tokens": 436, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-368", + "ok": true, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-85", + "ok": true, + "error": null, + "completion_tokens": 1329, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-246", + "ok": true, + "error": null, + "completion_tokens": 1399, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-23", + "ok": true, + "error": null, + "completion_tokens": 2899, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-372", + "ok": true, + "error": null, + "completion_tokens": 3022, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-387", + "ok": true, + "error": null, + "completion_tokens": 848, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-138", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-56", + "ok": true, + "error": null, + "completion_tokens": 1185, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-277", + "ok": true, + "error": null, + "completion_tokens": 1208, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-407", + "ok": false, + "error": null, + "completion_tokens": 1484, + "reasoning_chars": 0, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-171", + "ok": true, + "error": null, + "completion_tokens": 1755, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-96", + "ok": true, + "error": null, + "completion_tokens": 4316, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-134", + "ok": true, + "error": null, + "completion_tokens": 4195, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-218", + "ok": true, + "error": null, + "completion_tokens": 467, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-330", + "ok": true, + "error": null, + "completion_tokens": 3027, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-6", + "ok": true, + "error": null, + "completion_tokens": 1334, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-143", + "ok": true, + "error": null, + "completion_tokens": 379, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-121", + "ok": true, + "error": null, + "completion_tokens": 837, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-219", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-524", + "ok": true, + "error": null, + "completion_tokens": 387, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-206", + "ok": true, + "error": null, + "completion_tokens": 535, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-144", + "ok": true, + "error": null, + "completion_tokens": 1285, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-447", + "ok": true, + "error": null, + "completion_tokens": 966, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-492", + "ok": true, + "error": null, + "completion_tokens": 2542, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-485", + "ok": true, + "error": null, + "completion_tokens": 16872, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-220", + "ok": true, + "error": null, + "completion_tokens": 2617, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-412", + "ok": true, + "error": null, + "completion_tokens": 3010, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-342", + "ok": true, + "error": null, + "completion_tokens": 1605, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-222", + "ok": true, + "error": null, + "completion_tokens": 8999, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-131", + "ok": true, + "error": null, + "completion_tokens": 572, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-210", + "ok": true, + "error": null, + "completion_tokens": 392, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-137", + "ok": true, + "error": null, + "completion_tokens": 1179, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-464", + "ok": true, + "error": null, + "completion_tokens": 754, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-503", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-446", + "ok": true, + "error": null, + "completion_tokens": 792, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-2", + "ok": true, + "error": null, + "completion_tokens": 1046, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-525", + "ok": true, + "error": null, + "completion_tokens": 938, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-27", + "ok": true, + "error": null, + "completion_tokens": 929, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-136", + "ok": true, + "error": null, + "completion_tokens": 1668, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-286", + "ok": true, + "error": null, + "completion_tokens": 1053, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-449", + "ok": true, + "error": null, + "completion_tokens": 1217, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-55", + "ok": true, + "error": null, + "completion_tokens": 7033, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-93", + "ok": true, + "error": null, + "completion_tokens": 408, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-399", + "ok": true, + "error": null, + "completion_tokens": 1626, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-19", + "ok": true, + "error": null, + "completion_tokens": 200, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-229", + "ok": true, + "error": null, + "completion_tokens": 948, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-181", + "ok": true, + "error": null, + "completion_tokens": 507, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-362", + "ok": true, + "error": null, + "completion_tokens": 731, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-289", + "ok": true, + "error": null, + "completion_tokens": 380, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-356", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-115", + "ok": false, + "error": null, + "completion_tokens": 2552, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-404", + "ok": false, + "error": null, + "completion_tokens": 790, + "reasoning_chars": 0, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-243", + "ok": false, + "error": null, + "completion_tokens": 4281, + "reasoning_chars": 0, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-90", + "ok": true, + "error": null, + "completion_tokens": 2089, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-54", + "ok": true, + "error": null, + "completion_tokens": 1341, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-276", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-308", + "ok": true, + "error": null, + "completion_tokens": 1949, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-64", + "ok": false, + "error": null, + "completion_tokens": 1078, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-260", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-337", + "ok": true, + "error": null, + "completion_tokens": 1025, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-510", + "ok": true, + "error": null, + "completion_tokens": 2930, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-119", + "ok": true, + "error": null, + "completion_tokens": 1993, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-5", + "ok": true, + "error": null, + "completion_tokens": 336, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-468", + "ok": true, + "error": null, + "completion_tokens": 7435, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-418", + "ok": true, + "error": null, + "completion_tokens": 869, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-476", + "ok": true, + "error": null, + "completion_tokens": 3087, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-49", + "ok": true, + "error": null, + "completion_tokens": 5697, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-336", + "ok": true, + "error": null, + "completion_tokens": 283, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-422", + "ok": true, + "error": null, + "completion_tokens": 1249, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-417", + "ok": true, + "error": null, + "completion_tokens": 565, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-347", + "ok": true, + "error": null, + "completion_tokens": 10154, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-101", + "ok": true, + "error": null, + "completion_tokens": 554, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-209", + "ok": true, + "error": null, + "completion_tokens": 522, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-341", + "ok": true, + "error": null, + "completion_tokens": 4032, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-533", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-158", + "ok": true, + "error": null, + "completion_tokens": 782, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-321", + "ok": true, + "error": null, + "completion_tokens": 839, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-311", + "ok": true, + "error": null, + "completion_tokens": 191, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-50", + "ok": false, + "error": null, + "completion_tokens": 3485, + "reasoning_chars": 0, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-518", + "ok": true, + "error": null, + "completion_tokens": 3354, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-374", + "ok": true, + "error": null, + "completion_tokens": 338, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-317", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-512", + "ok": true, + "error": null, + "completion_tokens": 1811, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-102", + "ok": true, + "error": null, + "completion_tokens": 619, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-72", + "ok": true, + "error": null, + "completion_tokens": 1140, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-33", + "ok": true, + "error": null, + "completion_tokens": 1275, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-263", + "ok": true, + "error": null, + "completion_tokens": 576, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-501", + "ok": true, + "error": null, + "completion_tokens": 3900, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-4", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-444", + "ok": true, + "error": null, + "completion_tokens": 1287, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-228", + "ok": true, + "error": null, + "completion_tokens": 794, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-299", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-278", + "ok": true, + "error": null, + "completion_tokens": 5662, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-154", + "ok": true, + "error": null, + "completion_tokens": 822, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-301", + "ok": true, + "error": null, + "completion_tokens": 388, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-161", + "ok": true, + "error": null, + "completion_tokens": 3710, + "reasoning_chars": 0, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-394", + "ok": false, + "error": null, + "completion_tokens": 1415, + "reasoning_chars": 0, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-159", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 0, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + } + ] + } + } + }, + "original-w4a16": { + "started": "2026-10-10 13:48:58", + "finished": "2026-10-10 14:26:10", + "seed": 20260816, + "thinking": true, + "tasks": { + "mmlu": { + "task": "mmlu", + "n": 150, + "hits": 126, + "accuracy_pct": 84.0, + "accuracy_ex_trunc_pct": 84.0, + "truncated": 0, + "completion_tokens_p50": 243, + "completion_tokens_p90": 869, + "completion_tokens_max": 26968, + "reasoning_chars_p50": 824, + "reasoning_chars_p90": 2898, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1346.1, + "gen_tokens": 100578, + "tok_per_s": 74.7, + "details": [ + { + "id": "mmlu-12325", + "ok": true, + "error": null, + "completion_tokens": 639, + "reasoning_chars": 1213, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-154", + "ok": false, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 569, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12844", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1153", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 549, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11089", + "ok": true, + "error": null, + "completion_tokens": 915, + "reasoning_chars": 2824, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12782", + "ok": true, + "error": null, + "completion_tokens": 541, + "reasoning_chars": 1798, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5422", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12363", + "ok": true, + "error": null, + "completion_tokens": 923, + "reasoning_chars": 2467, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12556", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13163", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-270", + "ok": true, + "error": null, + "completion_tokens": 370, + "reasoning_chars": 929, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6943", + "ok": true, + "error": null, + "completion_tokens": 458, + "reasoning_chars": 1950, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8330", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9276", + "ok": true, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 331, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7584", + "ok": true, + "error": null, + "completion_tokens": 98, + "reasoning_chars": 322, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12819", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6163", + "ok": true, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 793, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4099", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 824, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10389", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 252, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1539", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12013", + "ok": true, + "error": null, + "completion_tokens": 572, + "reasoning_chars": 1715, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7814", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 868, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1106", + "ok": true, + "error": null, + "completion_tokens": 80, + "reasoning_chars": 278, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1616", + "ok": true, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 484, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-555", + "ok": true, + "error": null, + "completion_tokens": 450, + "reasoning_chars": 1115, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3698", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 1083, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-618", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 669, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9842", + "ok": true, + "error": null, + "completion_tokens": 111, + "reasoning_chars": 507, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4229", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 339, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1022", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 1050, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6421", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1997", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8941", + "ok": false, + "error": null, + "completion_tokens": 3824, + "reasoning_chars": 17117, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-565", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 964, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11364", + "ok": true, + "error": null, + "completion_tokens": 668, + "reasoning_chars": 1100, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5959", + "ok": true, + "error": null, + "completion_tokens": 588, + "reasoning_chars": 1400, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13066", + "ok": true, + "error": null, + "completion_tokens": 422, + "reasoning_chars": 1760, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3122", + "ok": true, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 433, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3722", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13839", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12570", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9959", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 418, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10791", + "ok": true, + "error": null, + "completion_tokens": 1398, + "reasoning_chars": 4971, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12268", + "ok": true, + "error": null, + "completion_tokens": 785, + "reasoning_chars": 2850, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9051", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 1132, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12887", + "ok": true, + "error": null, + "completion_tokens": 638, + "reasoning_chars": 2204, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2667", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 101, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3906", + "ok": true, + "error": null, + "completion_tokens": 366, + "reasoning_chars": 1149, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10304", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 527, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12927", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 286, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6337", + "ok": true, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 498, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3907", + "ok": true, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12888", + "ok": false, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 1354, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10629", + "ok": true, + "error": null, + "completion_tokens": 274, + "reasoning_chars": 482, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5801", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 639, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2936", + "ok": true, + "error": null, + "completion_tokens": 598, + "reasoning_chars": 1549, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7389", + "ok": true, + "error": null, + "completion_tokens": 348, + "reasoning_chars": 1315, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9532", + "ok": true, + "error": null, + "completion_tokens": 495, + "reasoning_chars": 1663, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8705", + "ok": true, + "error": null, + "completion_tokens": 465, + "reasoning_chars": 2215, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11776", + "ok": true, + "error": null, + "completion_tokens": 560, + "reasoning_chars": 2451, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6836", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 883, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7856", + "ok": true, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 509, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9697", + "ok": true, + "error": null, + "completion_tokens": 537, + "reasoning_chars": 1814, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10825", + "ok": true, + "error": null, + "completion_tokens": 1290, + "reasoning_chars": 4714, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11178", + "ok": true, + "error": null, + "completion_tokens": 708, + "reasoning_chars": 3552, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11420", + "ok": true, + "error": null, + "completion_tokens": 672, + "reasoning_chars": 2898, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4658", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5403", + "ok": false, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6160", + "ok": true, + "error": null, + "completion_tokens": 296, + "reasoning_chars": 1418, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6872", + "ok": false, + "error": null, + "completion_tokens": 26968, + "reasoning_chars": 115278, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9738", + "ok": true, + "error": null, + "completion_tokens": 290, + "reasoning_chars": 1139, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3076", + "ok": false, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13496", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 1022, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11289", + "ok": false, + "error": null, + "completion_tokens": 116, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-222", + "ok": true, + "error": null, + "completion_tokens": 355, + "reasoning_chars": 1142, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12180", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1143", + "ok": false, + "error": null, + "completion_tokens": 48, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13086", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7038", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13587", + "ok": true, + "error": null, + "completion_tokens": 666, + "reasoning_chars": 3602, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12349", + "ok": true, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 467, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11747", + "ok": true, + "error": null, + "completion_tokens": 539, + "reasoning_chars": 1547, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9848", + "ok": false, + "error": null, + "completion_tokens": 914, + "reasoning_chars": 3251, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3438", + "ok": true, + "error": null, + "completion_tokens": 316, + "reasoning_chars": 827, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-89", + "ok": true, + "error": null, + "completion_tokens": 245, + "reasoning_chars": 1008, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4253", + "ok": true, + "error": null, + "completion_tokens": 93, + "reasoning_chars": 314, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5460", + "ok": true, + "error": null, + "completion_tokens": 790, + "reasoning_chars": 2504, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2494", + "ok": false, + "error": null, + "completion_tokens": 9645, + "reasoning_chars": 31437, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11149", + "ok": false, + "error": null, + "completion_tokens": 1714, + "reasoning_chars": 6082, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-718", + "ok": true, + "error": null, + "completion_tokens": 412, + "reasoning_chars": 1182, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5330", + "ok": false, + "error": null, + "completion_tokens": 441, + "reasoning_chars": 1374, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10716", + "ok": true, + "error": null, + "completion_tokens": 4170, + "reasoning_chars": 16135, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5636", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 577, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11832", + "ok": true, + "error": null, + "completion_tokens": 646, + "reasoning_chars": 1515, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-95", + "ok": true, + "error": null, + "completion_tokens": 402, + "reasoning_chars": 1198, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13960", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 598, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2072", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 17, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12030", + "ok": true, + "error": null, + "completion_tokens": 1356, + "reasoning_chars": 5442, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2931", + "ok": false, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2441", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 294, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13666", + "ok": false, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6575", + "ok": true, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 562, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11625", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13283", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4262", + "ok": true, + "error": null, + "completion_tokens": 266, + "reasoning_chars": 581, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12931", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12150", + "ok": true, + "error": null, + "completion_tokens": 1015, + "reasoning_chars": 3242, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3678", + "ok": true, + "error": null, + "completion_tokens": 289, + "reasoning_chars": 1041, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4593", + "ok": true, + "error": null, + "completion_tokens": 561, + "reasoning_chars": 1535, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1563", + "ok": true, + "error": null, + "completion_tokens": 672, + "reasoning_chars": 3089, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13396", + "ok": true, + "error": null, + "completion_tokens": 502, + "reasoning_chars": 1332, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8049", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 740, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8396", + "ok": true, + "error": null, + "completion_tokens": 300, + "reasoning_chars": 1588, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8493", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 1001, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13514", + "ok": true, + "error": null, + "completion_tokens": 416, + "reasoning_chars": 1624, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3000", + "ok": true, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 120, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1635", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 514, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1391", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 543, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-627", + "ok": false, + "error": null, + "completion_tokens": 10126, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1573", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 813, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6731", + "ok": true, + "error": null, + "completion_tokens": 86, + "reasoning_chars": 352, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11665", + "ok": true, + "error": null, + "completion_tokens": 799, + "reasoning_chars": 3821, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2914", + "ok": true, + "error": null, + "completion_tokens": 467, + "reasoning_chars": 1599, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6875", + "ok": true, + "error": null, + "completion_tokens": 1184, + "reasoning_chars": 5395, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2184", + "ok": true, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 651, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12801", + "ok": true, + "error": null, + "completion_tokens": 289, + "reasoning_chars": 1432, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6717", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 529, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 639, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12464", + "ok": false, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2787", + "ok": true, + "error": null, + "completion_tokens": 398, + "reasoning_chars": 1111, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11549", + "ok": true, + "error": null, + "completion_tokens": 517, + "reasoning_chars": 1576, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-842", + "ok": true, + "error": null, + "completion_tokens": 869, + "reasoning_chars": 2466, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13868", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9647", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 954, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9502", + "ok": false, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12441", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 1380, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4392", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 292, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5036", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 672, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12606", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 1362, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1400", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 1083, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13979", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13634", + "ok": true, + "error": null, + "completion_tokens": 300, + "reasoning_chars": 897, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11226", + "ok": true, + "error": null, + "completion_tokens": 880, + "reasoning_chars": 2830, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-625", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-257", + "ok": true, + "error": null, + "completion_tokens": 228, + "reasoning_chars": 848, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12853", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 642, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-451", + "ok": true, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 1329, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2074", + "ok": true, + "error": null, + "completion_tokens": 56, + "reasoning_chars": 138, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9117", + "ok": true, + "error": null, + "completion_tokens": 376, + "reasoning_chars": 1272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10748", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "cmmlu": { + "task": "cmmlu", + "n": 150, + "hits": 127, + "accuracy_pct": 84.67, + "accuracy_ex_trunc_pct": 84.67, + "truncated": 0, + "completion_tokens_p50": 210, + "completion_tokens_p90": 933, + "completion_tokens_max": 19215, + "reasoning_chars_p50": 386, + "reasoning_chars_p90": 2015, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1397.6, + "gen_tokens": 82459, + "tok_per_s": 59.0, + "details": [ + { + "id": "cmmlu-chinese_teacher_qualification-63", + "ok": true, + "error": null, + "completion_tokens": 230, + "reasoning_chars": 366, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-54", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 386, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-91", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 59, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-260", + "ok": true, + "error": null, + "completion_tokens": 498, + "reasoning_chars": 2080, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-32", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 225, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-29", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 392, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-200", + "ok": false, + "error": null, + "completion_tokens": 2118, + "reasoning_chars": 3220, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-98", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 393, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-44", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-118", + "ok": true, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 565, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-116", + "ok": true, + "error": null, + "completion_tokens": 1191, + "reasoning_chars": 1667, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-43", + "ok": true, + "error": null, + "completion_tokens": 490, + "reasoning_chars": 669, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-145", + "ok": true, + "error": null, + "completion_tokens": 516, + "reasoning_chars": 2045, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-247", + "ok": true, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 354, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-224", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 148, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-26", + "ok": true, + "error": null, + "completion_tokens": 724, + "reasoning_chars": 1547, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-105", + "ok": true, + "error": null, + "completion_tokens": 989, + "reasoning_chars": 1277, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-15", + "ok": true, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 920, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-62", + "ok": false, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-nutrition-74", + "ok": true, + "error": null, + "completion_tokens": 178, + "reasoning_chars": 303, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-181", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 204, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-127", + "ok": true, + "error": null, + "completion_tokens": 503, + "reasoning_chars": 885, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_geography-107", + "ok": false, + "error": null, + "completion_tokens": 349, + "reasoning_chars": 634, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-18", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 598, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-82", + "ok": true, + "error": null, + "completion_tokens": 116, + "reasoning_chars": 499, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_food_culture-38", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 149, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-198", + "ok": true, + "error": null, + "completion_tokens": 95, + "reasoning_chars": 114, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-96", + "ok": true, + "error": null, + "completion_tokens": 71, + "reasoning_chars": 95, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-73", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-20", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-95", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 617, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-88", + "ok": true, + "error": null, + "completion_tokens": 1497, + "reasoning_chars": 6607, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-192", + "ok": true, + "error": null, + "completion_tokens": 276, + "reasoning_chars": 477, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-66", + "ok": true, + "error": null, + "completion_tokens": 395, + "reasoning_chars": 420, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-100", + "ok": true, + "error": null, + "completion_tokens": 310, + "reasoning_chars": 1288, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-136", + "ok": true, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 947, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-16", + "ok": false, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 1447, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-90", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 1093, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-41", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-345", + "ok": true, + "error": null, + "completion_tokens": 439, + "reasoning_chars": 1564, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-103", + "ok": false, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-93", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-88", + "ok": true, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-206", + "ok": true, + "error": null, + "completion_tokens": 254, + "reasoning_chars": 408, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-105", + "ok": true, + "error": null, + "completion_tokens": 686, + "reasoning_chars": 785, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-90", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 816, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-190", + "ok": false, + "error": null, + "completion_tokens": 571, + "reasoning_chars": 735, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-international_law-39", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 442, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-155", + "ok": true, + "error": null, + "completion_tokens": 684, + "reasoning_chars": 1144, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-24", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 335, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-6", + "ok": true, + "error": null, + "completion_tokens": 509, + "reasoning_chars": 1835, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-31", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-243", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 647, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-63", + "ok": false, + "error": null, + "completion_tokens": 1263, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-135", + "ok": true, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 29, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-89", + "ok": true, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 31, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-91", + "ok": false, + "error": null, + "completion_tokens": 659, + "reasoning_chars": 790, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-120", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 432, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-62", + "ok": true, + "error": null, + "completion_tokens": 226, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-106", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 154, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-food_science-106", + "ok": false, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 379, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-24", + "ok": true, + "error": null, + "completion_tokens": 679, + "reasoning_chars": 1164, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-157", + "ok": true, + "error": null, + "completion_tokens": 567, + "reasoning_chars": 729, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-41", + "ok": true, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 1526, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-115", + "ok": true, + "error": null, + "completion_tokens": 2004, + "reasoning_chars": 6243, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-journalism-29", + "ok": true, + "error": null, + "completion_tokens": 95, + "reasoning_chars": 142, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-67", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 129, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-161", + "ok": true, + "error": null, + "completion_tokens": 65, + "reasoning_chars": 65, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-120", + "ok": true, + "error": null, + "completion_tokens": 296, + "reasoning_chars": 1176, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-education-117", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 212, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-169", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 280, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-129", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-19", + "ok": true, + "error": null, + "completion_tokens": 1356, + "reasoning_chars": 5096, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-8", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 648, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-72", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 386, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-public_relations-139", + "ok": false, + "error": null, + "completion_tokens": 646, + "reasoning_chars": 1087, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_security-22", + "ok": true, + "error": null, + "completion_tokens": 205, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-7", + "ok": true, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 85, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-108", + "ok": true, + "error": null, + "completion_tokens": 1218, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-31", + "ok": true, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 28, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-131", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 446, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-35", + "ok": false, + "error": null, + "completion_tokens": 45, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_actuarial_science-38", + "ok": true, + "error": null, + "completion_tokens": 1903, + "reasoning_chars": 3614, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-165", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 368, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-101", + "ok": true, + "error": null, + "completion_tokens": 397, + "reasoning_chars": 677, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-107", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 129, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-8", + "ok": false, + "error": null, + "completion_tokens": 6143, + "reasoning_chars": 25187, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medical_statistics-86", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 837, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-security_study-30", + "ok": true, + "error": null, + "completion_tokens": 314, + "reasoning_chars": 520, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-118", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 934, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_law-24", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-46", + "ok": true, + "error": null, + "completion_tokens": 348, + "reasoning_chars": 626, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-anatomy-21", + "ok": false, + "error": null, + "completion_tokens": 486, + "reasoning_chars": 732, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-145", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 379, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-arts-91", + "ok": true, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 30, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-109", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 11, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-44", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 238, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-74", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 126, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-63", + "ok": true, + "error": null, + "completion_tokens": 71, + "reasoning_chars": 195, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-astronomy-146", + "ok": true, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-109", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 437, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-construction_project_management-16", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 174, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-113", + "ok": false, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 262, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-102", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 491, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-207", + "ok": false, + "error": null, + "completion_tokens": 972, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-19", + "ok": true, + "error": null, + "completion_tokens": 45, + "reasoning_chars": 196, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-164", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 293, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-338", + "ok": true, + "error": null, + "completion_tokens": 933, + "reasoning_chars": 2729, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-7", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 1247, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-92", + "ok": true, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 761, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-47", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 651, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-astronomy-140", + "ok": true, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 71, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-100", + "ok": true, + "error": null, + "completion_tokens": 101, + "reasoning_chars": 147, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-37", + "ok": true, + "error": null, + "completion_tokens": 764, + "reasoning_chars": 2015, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-191", + "ok": false, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 60, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-34", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 171, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-88", + "ok": true, + "error": null, + "completion_tokens": 266, + "reasoning_chars": 397, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-public_relations-77", + "ok": false, + "error": null, + "completion_tokens": 340, + "reasoning_chars": 1284, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-306", + "ok": false, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-40", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 366, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_chemistry-7", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-152", + "ok": true, + "error": null, + "completion_tokens": 423, + "reasoning_chars": 785, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-1", + "ok": true, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 53, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-94", + "ok": true, + "error": null, + "completion_tokens": 86, + "reasoning_chars": 128, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-109", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 379, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-164", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 94, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-89", + "ok": true, + "error": null, + "completion_tokens": 804, + "reasoning_chars": 3134, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-17", + "ok": true, + "error": null, + "completion_tokens": 599, + "reasoning_chars": 2188, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-arts-2", + "ok": true, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 183, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-51", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 943, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-141", + "ok": true, + "error": null, + "completion_tokens": 308, + "reasoning_chars": 1411, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-196", + "ok": true, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 69, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-97", + "ok": true, + "error": null, + "completion_tokens": 40, + "reasoning_chars": 42, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-202", + "ok": true, + "error": null, + "completion_tokens": 8714, + "reasoning_chars": 35180, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-195", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 264, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-21", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-49", + "ok": true, + "error": null, + "completion_tokens": 1434, + "reasoning_chars": 2802, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-94", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-8", + "ok": false, + "error": null, + "completion_tokens": 556, + "reasoning_chars": 734, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-98", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 233, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-28", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 466, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-74", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-121", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 409, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-120", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 387, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-105", + "ok": true, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 85, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-35", + "ok": true, + "error": null, + "completion_tokens": 19215, + "reasoning_chars": 52471, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-51", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 308, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-49", + "ok": true, + "error": null, + "completion_tokens": 1675, + "reasoning_chars": 4077, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-130", + "ok": true, + "error": null, + "completion_tokens": 452, + "reasoning_chars": 1460, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-140", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 867, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "mmlu_pro": { + "task": "mmlu_pro", + "n": 150, + "hits": 118, + "accuracy_pct": 78.67, + "accuracy_ex_trunc_pct": 80.27, + "truncated": 3, + "completion_tokens_p50": 271, + "completion_tokens_p90": 2190, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 759, + "reasoning_chars_p90": 4751, + "reasoning_leaked": null, + "empty": 3, + "length_limited": 3, + "errors": 0, + "elapsed_s": 1945.5, + "gen_tokens": 274989, + "tok_per_s": 141.3, + "details": [ + { + "id": "mmlupro-5831", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 773, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2299", + "ok": true, + "error": null, + "completion_tokens": 493, + "reasoning_chars": 1757, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9280", + "ok": true, + "error": null, + "completion_tokens": 2168, + "reasoning_chars": 6332, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5486", + "ok": true, + "error": null, + "completion_tokens": 643, + "reasoning_chars": 2819, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5761", + "ok": true, + "error": null, + "completion_tokens": 528, + "reasoning_chars": 1360, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6589", + "ok": true, + "error": null, + "completion_tokens": 834, + "reasoning_chars": 2378, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5389", + "ok": true, + "error": null, + "completion_tokens": 254, + "reasoning_chars": 737, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5131", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1677", + "ok": true, + "error": null, + "completion_tokens": 1594, + "reasoning_chars": 5852, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4388", + "ok": false, + "error": null, + "completion_tokens": 11381, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3285", + "ok": true, + "error": null, + "completion_tokens": 1080, + "reasoning_chars": 2989, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4118", + "ok": true, + "error": null, + "completion_tokens": 1277, + "reasoning_chars": 3665, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5344", + "ok": false, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11696", + "ok": true, + "error": null, + "completion_tokens": 3318, + "reasoning_chars": 11349, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-858", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 305, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-214", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 103617, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "mmlupro-6609", + "ok": true, + "error": null, + "completion_tokens": 920, + "reasoning_chars": 2411, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2083", + "ok": true, + "error": null, + "completion_tokens": 667, + "reasoning_chars": 2852, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9752", + "ok": false, + "error": null, + "completion_tokens": 21877, + "reasoning_chars": 66860, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7213", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5234", + "ok": true, + "error": null, + "completion_tokens": 90, + "reasoning_chars": 378, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8570", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 363, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5612", + "ok": true, + "error": null, + "completion_tokens": 111, + "reasoning_chars": 167, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2071", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9842", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 942, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4665", + "ok": true, + "error": null, + "completion_tokens": 1209, + "reasoning_chars": 2325, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5564", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6475", + "ok": true, + "error": null, + "completion_tokens": 69, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2153", + "ok": true, + "error": null, + "completion_tokens": 367, + "reasoning_chars": 1239, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5812", + "ok": true, + "error": null, + "completion_tokens": 137, + "reasoning_chars": 231, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3407", + "ok": true, + "error": null, + "completion_tokens": 3328, + "reasoning_chars": 3338, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2879", + "ok": false, + "error": null, + "completion_tokens": 1366, + "reasoning_chars": 4751, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8266", + "ok": false, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1916", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3486", + "ok": false, + "error": null, + "completion_tokens": 730, + "reasoning_chars": 2060, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11275", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 698, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6040", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8565", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 354, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6848", + "ok": true, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8926", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 386, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8458", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7854", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 951, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3962", + "ok": true, + "error": null, + "completion_tokens": 1344, + "reasoning_chars": 2786, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6470", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3813", + "ok": true, + "error": null, + "completion_tokens": 301, + "reasoning_chars": 788, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8026", + "ok": true, + "error": null, + "completion_tokens": 93, + "reasoning_chars": 201, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11030", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8151", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 15, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5286", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3888", + "ok": true, + "error": null, + "completion_tokens": 40, + "reasoning_chars": 127, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11676", + "ok": true, + "error": null, + "completion_tokens": 2835, + "reasoning_chars": 5659, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8762", + "ok": false, + "error": null, + "completion_tokens": 1026, + "reasoning_chars": 1912, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-12160", + "ok": true, + "error": null, + "completion_tokens": 548, + "reasoning_chars": 1403, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4022", + "ok": true, + "error": null, + "completion_tokens": 2507, + "reasoning_chars": 5832, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10092", + "ok": true, + "error": null, + "completion_tokens": 502, + "reasoning_chars": 1658, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7364", + "ok": true, + "error": null, + "completion_tokens": 326, + "reasoning_chars": 939, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6001", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10873", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9777", + "ok": true, + "error": null, + "completion_tokens": 2190, + "reasoning_chars": 4370, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8723", + "ok": true, + "error": null, + "completion_tokens": 200, + "reasoning_chars": 476, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4595", + "ok": true, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 762, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8423", + "ok": true, + "error": null, + "completion_tokens": 5591, + "reasoning_chars": 9333, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11051", + "ok": false, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 950, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-836", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 140, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11297", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 98267, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "mmlupro-3950", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 511, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1562", + "ok": false, + "error": null, + "completion_tokens": 1125, + "reasoning_chars": 4165, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2041", + "ok": true, + "error": null, + "completion_tokens": 564, + "reasoning_chars": 2698, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3473", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 231, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11253", + "ok": true, + "error": null, + "completion_tokens": 2027, + "reasoning_chars": 8852, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3858", + "ok": true, + "error": null, + "completion_tokens": 789, + "reasoning_chars": 1627, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2231", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2190", + "ok": true, + "error": null, + "completion_tokens": 399, + "reasoning_chars": 1143, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6930", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 1790, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6558", + "ok": true, + "error": null, + "completion_tokens": 718, + "reasoning_chars": 2019, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5371", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1766", + "ok": true, + "error": null, + "completion_tokens": 945, + "reasoning_chars": 3207, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6045", + "ok": true, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2298", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6400", + "ok": true, + "error": null, + "completion_tokens": 208, + "reasoning_chars": 839, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6499", + "ok": true, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7936", + "ok": true, + "error": null, + "completion_tokens": 388, + "reasoning_chars": 1531, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8566", + "ok": true, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 247, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11633", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 1179, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9843", + "ok": false, + "error": null, + "completion_tokens": 10732, + "reasoning_chars": 26003, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11758", + "ok": false, + "error": null, + "completion_tokens": 19857, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2843", + "ok": true, + "error": null, + "completion_tokens": 1214, + "reasoning_chars": 3219, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7483", + "ok": false, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11379", + "ok": true, + "error": null, + "completion_tokens": 1580, + "reasoning_chars": 4275, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4866", + "ok": true, + "error": null, + "completion_tokens": 775, + "reasoning_chars": 2382, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-818", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 386, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2785", + "ok": true, + "error": null, + "completion_tokens": 943, + "reasoning_chars": 4093, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8119", + "ok": true, + "error": null, + "completion_tokens": 487, + "reasoning_chars": 1648, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2214", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5774", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6049", + "ok": false, + "error": null, + "completion_tokens": 526, + "reasoning_chars": 1441, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7219", + "ok": true, + "error": null, + "completion_tokens": 438, + "reasoning_chars": 1548, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1660", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4767", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5964", + "ok": false, + "error": null, + "completion_tokens": 414, + "reasoning_chars": 1122, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2743", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 1198, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4062", + "ok": true, + "error": null, + "completion_tokens": 296, + "reasoning_chars": 680, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7368", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 638, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-827", + "ok": false, + "error": null, + "completion_tokens": 503, + "reasoning_chars": 2021, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11877", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 71786, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "mmlupro-7557", + "ok": true, + "error": null, + "completion_tokens": 554, + "reasoning_chars": 1624, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9998", + "ok": false, + "error": null, + "completion_tokens": 86, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10098", + "ok": false, + "error": null, + "completion_tokens": 914, + "reasoning_chars": 3789, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1744", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9210", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 460, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11248", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 741, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11187", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 125, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2799", + "ok": true, + "error": null, + "completion_tokens": 283, + "reasoning_chars": 1321, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4479", + "ok": true, + "error": null, + "completion_tokens": 2903, + "reasoning_chars": 6282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10223", + "ok": true, + "error": null, + "completion_tokens": 634, + "reasoning_chars": 1257, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5698", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 524, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8291", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 353, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3516", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5466", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4851", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-733", + "ok": true, + "error": null, + "completion_tokens": 1197, + "reasoning_chars": 1597, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6995", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 468, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2573", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8481", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 655, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5938", + "ok": true, + "error": null, + "completion_tokens": 756, + "reasoning_chars": 2804, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10708", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8476", + "ok": true, + "error": null, + "completion_tokens": 364, + "reasoning_chars": 879, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7990", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 748, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2409", + "ok": true, + "error": null, + "completion_tokens": 510, + "reasoning_chars": 2456, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11551", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-674", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 270, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1978", + "ok": true, + "error": null, + "completion_tokens": 973, + "reasoning_chars": 3227, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10420", + "ok": false, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3445", + "ok": true, + "error": null, + "completion_tokens": 838, + "reasoning_chars": 2751, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9488", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 515, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4993", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 446, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4316", + "ok": true, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 267, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4730", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11570", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 903, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2149", + "ok": false, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8322", + "ok": true, + "error": null, + "completion_tokens": 1093, + "reasoning_chars": 1905, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10993", + "ok": false, + "error": null, + "completion_tokens": 503, + "reasoning_chars": 1922, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4588", + "ok": true, + "error": null, + "completion_tokens": 1594, + "reasoning_chars": 2938, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3258", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8629", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 483, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-637", + "ok": true, + "error": null, + "completion_tokens": 314, + "reasoning_chars": 724, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4257", + "ok": true, + "error": null, + "completion_tokens": 275, + "reasoning_chars": 759, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9719", + "ok": true, + "error": null, + "completion_tokens": 1348, + "reasoning_chars": 3507, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11714", + "ok": true, + "error": null, + "completion_tokens": 24837, + "reasoning_chars": 57879, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9678", + "ok": false, + "error": null, + "completion_tokens": 12445, + "reasoning_chars": 57196, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "ceval": { + "task": "ceval", + "n": 150, + "hits": 126, + "accuracy_pct": 84.0, + "accuracy_ex_trunc_pct": 84.56, + "truncated": 1, + "completion_tokens_p50": 240, + "completion_tokens_p90": 1291, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 390, + "reasoning_chars_p90": 3986, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 1484.2, + "gen_tokens": 193631, + "tok_per_s": 130.5, + "details": [ + { + "id": "ceval-high_school_chinese-1", + "ok": false, + "error": null, + "completion_tokens": 4988, + "reasoning_chars": 15262, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-10", + "ok": true, + "error": null, + "completion_tokens": 537, + "reasoning_chars": 797, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-41", + "ok": false, + "error": null, + "completion_tokens": 6145, + "reasoning_chars": 18133, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_physics-3", + "ok": true, + "error": null, + "completion_tokens": 283, + "reasoning_chars": 1185, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-21", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 274, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-4", + "ok": true, + "error": null, + "completion_tokens": 1898, + "reasoning_chars": 4277, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-21", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 288, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-2", + "ok": true, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 90, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-6", + "ok": true, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-12", + "ok": true, + "error": null, + "completion_tokens": 7155, + "reasoning_chars": 23431, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-19", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 198, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_chemistry-9", + "ok": true, + "error": null, + "completion_tokens": 645, + "reasoning_chars": 2268, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-15", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-22", + "ok": false, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 596, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-16", + "ok": true, + "error": null, + "completion_tokens": 40, + "reasoning_chars": 51, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-11", + "ok": true, + "error": null, + "completion_tokens": 2876, + "reasoning_chars": 4472, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-18", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 152, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-24", + "ok": true, + "error": null, + "completion_tokens": 515, + "reasoning_chars": 1481, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-29", + "ok": true, + "error": null, + "completion_tokens": 1389, + "reasoning_chars": 5697, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-2", + "ok": true, + "error": null, + "completion_tokens": 276, + "reasoning_chars": 264, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-11", + "ok": true, + "error": null, + "completion_tokens": 119, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-18", + "ok": true, + "error": null, + "completion_tokens": 493, + "reasoning_chars": 1989, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-0", + "ok": false, + "error": null, + "completion_tokens": 892, + "reasoning_chars": 2842, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-14", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-27", + "ok": true, + "error": null, + "completion_tokens": 1225, + "reasoning_chars": 4144, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-art_studies-19", + "ok": false, + "error": null, + "completion_tokens": 895, + "reasoning_chars": 1165, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-21", + "ok": true, + "error": null, + "completion_tokens": 4073, + "reasoning_chars": 16554, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-33", + "ok": true, + "error": null, + "completion_tokens": 145, + "reasoning_chars": 249, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-20", + "ok": true, + "error": null, + "completion_tokens": 274, + "reasoning_chars": 1204, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-10", + "ok": true, + "error": null, + "completion_tokens": 532, + "reasoning_chars": 2474, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-19", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 245, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_network-7", + "ok": true, + "error": null, + "completion_tokens": 116, + "reasoning_chars": 174, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-41", + "ok": true, + "error": null, + "completion_tokens": 639, + "reasoning_chars": 1946, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-44", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 320, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-9", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 445, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-28", + "ok": true, + "error": null, + "completion_tokens": 277, + "reasoning_chars": 1380, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-39", + "ok": true, + "error": null, + "completion_tokens": 792, + "reasoning_chars": 2528, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-1", + "ok": false, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 1058, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-24", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 186, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-5", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_architecture-15", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-39", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-23", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 283, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-7", + "ok": false, + "error": null, + "completion_tokens": 228, + "reasoning_chars": 359, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-17", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 368, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_chemistry-6", + "ok": true, + "error": null, + "completion_tokens": 341, + "reasoning_chars": 852, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-15", + "ok": true, + "error": null, + "completion_tokens": 261, + "reasoning_chars": 389, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-4", + "ok": true, + "error": null, + "completion_tokens": 12335, + "reasoning_chars": 34986, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-5", + "ok": true, + "error": null, + "completion_tokens": 723, + "reasoning_chars": 2832, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-50", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 387, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_politics-11", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 148, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-32", + "ok": true, + "error": null, + "completion_tokens": 595, + "reasoning_chars": 2095, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-16", + "ok": true, + "error": null, + "completion_tokens": 357, + "reasoning_chars": 1086, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-0", + "ok": true, + "error": null, + "completion_tokens": 272, + "reasoning_chars": 455, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-sports_science-9", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-art_studies-15", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 440, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-2", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 60192, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "ceval-fire_engineer-7", + "ok": true, + "error": null, + "completion_tokens": 1189, + "reasoning_chars": 1728, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-11", + "ok": true, + "error": null, + "completion_tokens": 367, + "reasoning_chars": 1363, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-operating_system-18", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 142, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-4", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-17", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-1", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-32", + "ok": true, + "error": null, + "completion_tokens": 69, + "reasoning_chars": 110, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-33", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-25", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 359, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-40", + "ok": true, + "error": null, + "completion_tokens": 257, + "reasoning_chars": 389, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-7", + "ok": true, + "error": null, + "completion_tokens": 1291, + "reasoning_chars": 4170, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-5", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-12", + "ok": true, + "error": null, + "completion_tokens": 192, + "reasoning_chars": 347, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-4", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 214, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-38", + "ok": false, + "error": null, + "completion_tokens": 28756, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-22", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-0", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-15", + "ok": false, + "error": null, + "completion_tokens": 18399, + "reasoning_chars": 44674, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-45", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 60, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-6", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 868, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-20", + "ok": true, + "error": null, + "completion_tokens": 499, + "reasoning_chars": 722, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-basic_medicine-2", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 448, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-17", + "ok": true, + "error": null, + "completion_tokens": 966, + "reasoning_chars": 3483, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-16", + "ok": false, + "error": null, + "completion_tokens": 628, + "reasoning_chars": 2574, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-7", + "ok": true, + "error": null, + "completion_tokens": 269, + "reasoning_chars": 695, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_biology-16", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-10", + "ok": true, + "error": null, + "completion_tokens": 250, + "reasoning_chars": 772, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-32", + "ok": true, + "error": null, + "completion_tokens": 3243, + "reasoning_chars": 8911, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-0", + "ok": true, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 112, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-12", + "ok": true, + "error": null, + "completion_tokens": 1248, + "reasoning_chars": 4282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-14", + "ok": true, + "error": null, + "completion_tokens": 473, + "reasoning_chars": 2272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-26", + "ok": true, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 456, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-3", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 16, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-3", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 620, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_network-13", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 198, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-sports_science-5", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 309, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-27", + "ok": true, + "error": null, + "completion_tokens": 769, + "reasoning_chars": 3089, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-21", + "ok": false, + "error": null, + "completion_tokens": 17476, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-10", + "ok": true, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 137, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-16", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-49", + "ok": true, + "error": null, + "completion_tokens": 300, + "reasoning_chars": 1100, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-13", + "ok": false, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 1275, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-5", + "ok": true, + "error": null, + "completion_tokens": 7158, + "reasoning_chars": 22946, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-2", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 74, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-27", + "ok": false, + "error": null, + "completion_tokens": 919, + "reasoning_chars": 1659, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_biology-8", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_physics-14", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-14", + "ok": true, + "error": null, + "completion_tokens": 187, + "reasoning_chars": 268, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-31", + "ok": true, + "error": null, + "completion_tokens": 555, + "reasoning_chars": 919, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-basic_medicine-13", + "ok": true, + "error": null, + "completion_tokens": 434, + "reasoning_chars": 1503, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-13", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-12", + "ok": true, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 830, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-44", + "ok": false, + "error": null, + "completion_tokens": 625, + "reasoning_chars": 642, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-3", + "ok": true, + "error": null, + "completion_tokens": 1036, + "reasoning_chars": 2740, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-13", + "ok": true, + "error": null, + "completion_tokens": 6007, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_mathematics-6", + "ok": true, + "error": null, + "completion_tokens": 86, + "reasoning_chars": 203, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_politics-19", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 826, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-environmental_impact_assessment_engineer-27", + "ok": false, + "error": null, + "completion_tokens": 595, + "reasoning_chars": 873, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-10", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 293, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-0", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-3", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 295, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-6", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 393, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_chinese-13", + "ok": true, + "error": null, + "completion_tokens": 298, + "reasoning_chars": 416, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-43", + "ok": false, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 390, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-42", + "ok": true, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 23, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-13", + "ok": true, + "error": null, + "completion_tokens": 296, + "reasoning_chars": 1215, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-discrete_mathematics-1", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 194, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-11", + "ok": true, + "error": null, + "completion_tokens": 579, + "reasoning_chars": 2510, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 269, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-2", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-1", + "ok": true, + "error": null, + "completion_tokens": 422, + "reasoning_chars": 957, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_physics-16", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-11", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 314, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-5", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 418, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-40", + "ok": true, + "error": null, + "completion_tokens": 230, + "reasoning_chars": 1129, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-6", + "ok": false, + "error": null, + "completion_tokens": 363, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_mathematics-10", + "ok": true, + "error": null, + "completion_tokens": 98, + "reasoning_chars": 229, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-10", + "ok": true, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-17", + "ok": false, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-11", + "ok": true, + "error": null, + "completion_tokens": 1113, + "reasoning_chars": 3986, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-16", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-1", + "ok": false, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 473, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_physics-0", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 410, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-37", + "ok": true, + "error": null, + "completion_tokens": 116, + "reasoning_chars": 193, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-9", + "ok": true, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 250, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 272, + "reasoning_chars": 391, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-5", + "ok": true, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 26, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-18", + "ok": true, + "error": null, + "completion_tokens": 485, + "reasoning_chars": 1170, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-30", + "ok": true, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 194, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-21", + "ok": true, + "error": null, + "completion_tokens": 264, + "reasoning_chars": 423, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-42", + "ok": true, + "error": null, + "completion_tokens": 360, + "reasoning_chars": 1199, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-5", + "ok": true, + "error": null, + "completion_tokens": 801, + "reasoning_chars": 2872, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_chemistry-4", + "ok": true, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 668, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "arc": { + "task": "arc", + "n": 150, + "hits": 121, + "accuracy_pct": 80.67, + "accuracy_ex_trunc_pct": 80.67, + "truncated": 0, + "completion_tokens_p50": 135, + "completion_tokens_p90": 437, + "completion_tokens_max": 1740, + "reasoning_chars_p50": 519, + "reasoning_chars_p90": 1322, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1383.3, + "gen_tokens": 30534, + "tok_per_s": 22.1, + "details": [ + { + "id": "arc-Mercury_415084", + "ok": true, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 231, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7249708", + "ok": true, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 1322, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_4_24", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7056823", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 721, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415534", + "ok": true, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 461, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405827", + "ok": false, + "error": null, + "completion_tokens": 1740, + "reasoning_chars": 7826, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7263183", + "ok": true, + "error": null, + "completion_tokens": 166, + "reasoning_chars": 509, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7106960", + "ok": true, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 190, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_7", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 561, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7182245", + "ok": true, + "error": null, + "completion_tokens": 485, + "reasoning_chars": 1779, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416177", + "ok": true, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-ACTAAP_2011_5_8", + "ok": true, + "error": null, + "completion_tokens": 336, + "reasoning_chars": 948, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_8_36", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 140, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401699", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 317, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7107240", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 207, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_4_26", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 473, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402638", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7026723", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415422", + "ok": true, + "error": null, + "completion_tokens": 296, + "reasoning_chars": 1010, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2000_8_2", + "ok": true, + "error": null, + "completion_tokens": 557, + "reasoning_chars": 1264, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2006_9_4", + "ok": true, + "error": null, + "completion_tokens": 283, + "reasoning_chars": 1071, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401272", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416169", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 372, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7116235", + "ok": true, + "error": null, + "completion_tokens": 437, + "reasoning_chars": 1263, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7109463", + "ok": true, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 809, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7032428", + "ok": true, + "error": null, + "completion_tokens": 683, + "reasoning_chars": 2189, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AIMS_2008_8_11", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 470, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_410334", + "ok": true, + "error": null, + "completion_tokens": 329, + "reasoning_chars": 1098, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_36", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415719", + "ok": true, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 305, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_17", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 1073, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402256", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 366, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_400443", + "ok": true, + "error": null, + "completion_tokens": 101, + "reasoning_chars": 338, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7133245", + "ok": true, + "error": null, + "completion_tokens": 308, + "reasoning_chars": 1039, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413458", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401372", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 820, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401116", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP__5_10312", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 415, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2007_4_10280", + "ok": true, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 271, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_416683", + "ok": true, + "error": null, + "completion_tokens": 544, + "reasoning_chars": 1375, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7190120", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 579, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401295", + "ok": true, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 261, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_4_28", + "ok": true, + "error": null, + "completion_tokens": 48, + "reasoning_chars": 197, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7223493", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 519, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7018410", + "ok": true, + "error": null, + "completion_tokens": 172, + "reasoning_chars": 525, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1998_8_24", + "ok": true, + "error": null, + "completion_tokens": 692, + "reasoning_chars": 1795, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_31", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 925, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405501", + "ok": true, + "error": null, + "completion_tokens": 277, + "reasoning_chars": 994, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408030", + "ok": true, + "error": null, + "completion_tokens": 340, + "reasoning_chars": 854, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401684", + "ok": false, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7041668", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7205870", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 843, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7210140", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 1168, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_405873", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 1187, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400052", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7018270", + "ok": true, + "error": null, + "completion_tokens": 406, + "reasoning_chars": 1326, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_406550", + "ok": true, + "error": null, + "completion_tokens": 250, + "reasoning_chars": 800, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7150728", + "ok": true, + "error": null, + "completion_tokens": 277, + "reasoning_chars": 1261, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400839", + "ok": true, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 518, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_30", + "ok": true, + "error": null, + "completion_tokens": 95, + "reasoning_chars": 399, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400383", + "ok": true, + "error": null, + "completion_tokens": 495, + "reasoning_chars": 1351, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7236740", + "ok": true, + "error": null, + "completion_tokens": 212, + "reasoning_chars": 733, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-CSZ_2005_5_CSZ10021", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 465, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7093958", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 1025, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7026355", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408324", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 432, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP__7_10348", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 278, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402629", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 258, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400046", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 977, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7180810", + "ok": true, + "error": null, + "completion_tokens": 523, + "reasoning_chars": 1378, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_180005", + "ok": true, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 414, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AIMS_2009_8_14", + "ok": true, + "error": null, + "completion_tokens": 293, + "reasoning_chars": 1222, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2007_5_16", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_LBS10662", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 607, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_23", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 445, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2012_8_19", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7041650", + "ok": true, + "error": null, + "completion_tokens": 350, + "reasoning_chars": 808, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416181", + "ok": true, + "error": null, + "completion_tokens": 95, + "reasoning_chars": 410, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TIMSS_1995_8_J6", + "ok": true, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 978, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2014_5_13", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 943, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7106750", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 541, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1999_4_23", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 760, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MEA_2013_5_12", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 228, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_8_42", + "ok": false, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 460, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408367", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 417, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7250058", + "ok": true, + "error": null, + "completion_tokens": 393, + "reasoning_chars": 940, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_414041", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 969, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401290", + "ok": true, + "error": null, + "completion_tokens": 395, + "reasoning_chars": 1038, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_409295", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 477, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7211680", + "ok": true, + "error": null, + "completion_tokens": 447, + "reasoning_chars": 1399, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_37", + "ok": true, + "error": null, + "completion_tokens": 377, + "reasoning_chars": 1114, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413439", + "ok": true, + "error": null, + "completion_tokens": 394, + "reasoning_chars": 1210, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_32", + "ok": true, + "error": null, + "completion_tokens": 51, + "reasoning_chars": 205, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7216423", + "ok": true, + "error": null, + "completion_tokens": 343, + "reasoning_chars": 1055, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-ACTAAP_2015_7_9", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7230405", + "ok": true, + "error": null, + "completion_tokens": 513, + "reasoning_chars": 1460, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7025008", + "ok": true, + "error": null, + "completion_tokens": 489, + "reasoning_chars": 1263, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_416536", + "ok": true, + "error": null, + "completion_tokens": 76, + "reasoning_chars": 336, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TIMSS_2007_8_pg60", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 556, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2009_4_34", + "ok": true, + "error": null, + "completion_tokens": 301, + "reasoning_chars": 1334, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_406833", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 953, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_409647", + "ok": true, + "error": null, + "completion_tokens": 466, + "reasoning_chars": 1398, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_184363", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7107170", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 707, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416104", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_10-v1", + "ok": true, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7202843", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7029803", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 478, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7164623", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 897, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_26", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7263060", + "ok": true, + "error": null, + "completion_tokens": 440, + "reasoning_chars": 1189, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7268328", + "ok": true, + "error": null, + "completion_tokens": 102, + "reasoning_chars": 424, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402282", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 720, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7029558", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 144, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_4_21", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7186358", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 1173, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2011_8_10434", + "ok": true, + "error": null, + "completion_tokens": 437, + "reasoning_chars": 1214, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2012_8_23651", + "ok": true, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 793, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_407125", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 648, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7268258", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7027405", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 484, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_8_7", + "ok": true, + "error": null, + "completion_tokens": 495, + "reasoning_chars": 2375, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_191625", + "ok": false, + "error": null, + "completion_tokens": 405, + "reasoning_chars": 1363, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_6", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 216, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7219678", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 849, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401678", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 748, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401312", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_8_42", + "ok": false, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 184, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7030468", + "ok": true, + "error": null, + "completion_tokens": 71, + "reasoning_chars": 315, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_175385", + "ok": false, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_175560", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_5", + "ok": true, + "error": null, + "completion_tokens": 88, + "reasoning_chars": 376, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_409009", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7128923", + "ok": false, + "error": null, + "completion_tokens": 332, + "reasoning_chars": 1482, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_8_9", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7042945", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 1163, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_8_47", + "ok": true, + "error": null, + "completion_tokens": 145, + "reasoning_chars": 689, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7185343", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 980, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7218400", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 499, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7221988", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_405950", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 761, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7222530", + "ok": true, + "error": null, + "completion_tokens": 406, + "reasoning_chars": 1188, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7102323", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2009_5_39", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 430, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1999_8_5", + "ok": true, + "error": null, + "completion_tokens": 339, + "reasoning_chars": 952, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400661", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7081603", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 647, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2008_3_25", + "ok": false, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413628", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_406427", + "ok": true, + "error": null, + "completion_tokens": 465, + "reasoning_chars": 1151, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "truthfulqa": { + "task": "truthfulqa", + "n": 150, + "hits": 88, + "accuracy_pct": 58.67, + "accuracy_ex_trunc_pct": 58.67, + "truncated": 0, + "completion_tokens_p50": 27, + "completion_tokens_p90": 445, + "completion_tokens_max": 980, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 1322, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1402.0, + "gen_tokens": 22891, + "tok_per_s": 16.3, + "details": [ + { + "id": "tqa-210", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-76", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-461", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-218", + "ok": true, + "error": null, + "completion_tokens": 124, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-254", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-635", + "ok": true, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 861, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-193", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 75, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-148", + "ok": false, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-400", + "ok": false, + "error": null, + "completion_tokens": 388, + "reasoning_chars": 1243, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-766", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-240", + "ok": false, + "error": null, + "completion_tokens": 898, + "reasoning_chars": 3291, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-50", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-83", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-381", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-325", + "ok": false, + "error": null, + "completion_tokens": 256, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-655", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-721", + "ok": true, + "error": null, + "completion_tokens": 364, + "reasoning_chars": 977, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-391", + "ok": true, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-401", + "ok": true, + "error": null, + "completion_tokens": 406, + "reasoning_chars": 1188, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-625", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 941, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-80", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-439", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-66", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-155", + "ok": true, + "error": null, + "completion_tokens": 445, + "reasoning_chars": 1109, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-516", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-522", + "ok": false, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-47", + "ok": true, + "error": null, + "completion_tokens": 345, + "reasoning_chars": 1593, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-343", + "ok": false, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-126", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-77", + "ok": false, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-613", + "ok": true, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 98, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-590", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 902, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-674", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-438", + "ok": false, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 402, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-503", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-639", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-111", + "ok": true, + "error": null, + "completion_tokens": 639, + "reasoning_chars": 2144, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-289", + "ok": true, + "error": null, + "completion_tokens": 435, + "reasoning_chars": 1310, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-705", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-373", + "ok": true, + "error": null, + "completion_tokens": 533, + "reasoning_chars": 1562, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-363", + "ok": true, + "error": null, + "completion_tokens": 583, + "reasoning_chars": 1743, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-792", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-113", + "ok": false, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 591, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-74", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 662, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-634", + "ok": true, + "error": null, + "completion_tokens": 508, + "reasoning_chars": 1395, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-804", + "ok": true, + "error": null, + "completion_tokens": 406, + "reasoning_chars": 1428, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-626", + "ok": true, + "error": null, + "completion_tokens": 310, + "reasoning_chars": 957, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-676", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-174", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-455", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 598, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-725", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-296", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-535", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-172", + "ok": true, + "error": null, + "completion_tokens": 335, + "reasoning_chars": 956, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-23", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-43", + "ok": true, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-38", + "ok": true, + "error": null, + "completion_tokens": 606, + "reasoning_chars": 1766, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-97", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-657", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-814", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-346", + "ok": true, + "error": null, + "completion_tokens": 252, + "reasoning_chars": 834, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-365", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-384", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-677", + "ok": true, + "error": null, + "completion_tokens": 505, + "reasoning_chars": 1521, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-486", + "ok": true, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-55", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-680", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 673, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-510", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-128", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 807, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-805", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-394", + "ok": true, + "error": null, + "completion_tokens": 472, + "reasoning_chars": 1348, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-435", + "ok": true, + "error": null, + "completion_tokens": 309, + "reasoning_chars": 890, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-285", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-687", + "ok": true, + "error": null, + "completion_tokens": 417, + "reasoning_chars": 1210, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-588", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-100", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-399", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-266", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-693", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 668, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-532", + "ok": true, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-558", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 731, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-319", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-459", + "ok": true, + "error": null, + "completion_tokens": 351, + "reasoning_chars": 1035, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-675", + "ok": true, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-129", + "ok": true, + "error": null, + "completion_tokens": 821, + "reasoning_chars": 3320, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-331", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-12", + "ok": false, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-164", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-306", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 525, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-275", + "ok": false, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-418", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-264", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-94", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-389", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-427", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-269", + "ok": false, + "error": null, + "completion_tokens": 72, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-643", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-7", + "ok": true, + "error": null, + "completion_tokens": 442, + "reasoning_chars": 1166, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-419", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-616", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-699", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-473", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-523", + "ok": true, + "error": null, + "completion_tokens": 571, + "reasoning_chars": 1946, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-214", + "ok": false, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-376", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 675, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-123", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-200", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-493", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-474", + "ok": true, + "error": null, + "completion_tokens": 309, + "reasoning_chars": 655, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-387", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-746", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-790", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-181", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-413", + "ok": false, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-349", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-434", + "ok": true, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-601", + "ok": true, + "error": null, + "completion_tokens": 321, + "reasoning_chars": 940, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-489", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-286", + "ok": false, + "error": null, + "completion_tokens": 286, + "reasoning_chars": 799, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-488", + "ok": true, + "error": null, + "completion_tokens": 460, + "reasoning_chars": 1322, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-759", + "ok": true, + "error": null, + "completion_tokens": 316, + "reasoning_chars": 769, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-273", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 755, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-59", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-315", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-425", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-58", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-691", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-497", + "ok": false, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-515", + "ok": false, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-398", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-670", + "ok": true, + "error": null, + "completion_tokens": 980, + "reasoning_chars": 1904, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-170", + "ok": false, + "error": null, + "completion_tokens": 463, + "reasoning_chars": 1174, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-334", + "ok": true, + "error": null, + "completion_tokens": 643, + "reasoning_chars": 1895, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-307", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-495", + "ok": true, + "error": null, + "completion_tokens": 351, + "reasoning_chars": 1166, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-597", + "ok": false, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-775", + "ok": true, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-186", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-733", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-308", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-5", + "ok": false, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-231", + "ok": true, + "error": null, + "completion_tokens": 273, + "reasoning_chars": 763, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-356", + "ok": false, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-810", + "ok": true, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-270", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-13", + "ok": false, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 238, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-114", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 914, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-4", + "ok": true, + "error": null, + "completion_tokens": 475, + "reasoning_chars": 1382, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-417", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 579, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-750", + "ok": false, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "gsm8k": { + "task": "gsm8k", + "n": 100, + "hits": 95, + "accuracy_pct": 95.0, + "accuracy_ex_trunc_pct": 95.0, + "truncated": 0, + "completion_tokens_p50": 192, + "completion_tokens_p90": 624, + "completion_tokens_max": 5481, + "reasoning_chars_p50": 365, + "reasoning_chars_p90": 1717, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1417.1, + "gen_tokens": 35938, + "tok_per_s": 25.4, + "details": [ + { + "id": "gsm8k-144", + "ok": true, + "error": null, + "completion_tokens": 308, + "reasoning_chars": 900, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-627", + "ok": true, + "error": null, + "completion_tokens": 276, + "reasoning_chars": 565, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-195", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 339, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-201", + "ok": true, + "error": null, + "completion_tokens": 83, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-730", + "ok": true, + "error": null, + "completion_tokens": 131, + "reasoning_chars": 251, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-166", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 364, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1141", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 270, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-832", + "ok": true, + "error": null, + "completion_tokens": 679, + "reasoning_chars": 1962, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-53", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 526, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1042", + "ok": false, + "error": null, + "completion_tokens": 988, + "reasoning_chars": 2939, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-609", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 275, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-318", + "ok": true, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-892", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 350, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-204", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 235, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1120", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 508, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1040", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 637, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-556", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1015", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 365, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-206", + "ok": true, + "error": null, + "completion_tokens": 844, + "reasoning_chars": 2713, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1058", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1044", + "ok": true, + "error": null, + "completion_tokens": 192, + "reasoning_chars": 419, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-131", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 329, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1154", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 297, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-158", + "ok": true, + "error": null, + "completion_tokens": 93, + "reasoning_chars": 225, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1149", + "ok": true, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 298, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1156", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 317, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-60", + "ok": true, + "error": null, + "completion_tokens": 252, + "reasoning_chars": 686, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-918", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1010", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 526, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1032", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 802, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1053", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 433, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-577", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 301, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-935", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 438, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-692", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-153", + "ok": true, + "error": null, + "completion_tokens": 536, + "reasoning_chars": 1354, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-40", + "ok": true, + "error": null, + "completion_tokens": 700, + "reasoning_chars": 2320, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-613", + "ok": true, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-875", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 441, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-189", + "ok": true, + "error": null, + "completion_tokens": 1161, + "reasoning_chars": 3222, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-800", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 256, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-169", + "ok": true, + "error": null, + "completion_tokens": 192, + "reasoning_chars": 436, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1169", + "ok": true, + "error": null, + "completion_tokens": 624, + "reasoning_chars": 1705, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-211", + "ok": true, + "error": null, + "completion_tokens": 257, + "reasoning_chars": 335, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1183", + "ok": true, + "error": null, + "completion_tokens": 4513, + "reasoning_chars": 17506, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-193", + "ok": true, + "error": null, + "completion_tokens": 338, + "reasoning_chars": 849, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-230", + "ok": true, + "error": null, + "completion_tokens": 339, + "reasoning_chars": 456, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-787", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 384, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-83", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 170, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1318", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 261, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-733", + "ok": true, + "error": null, + "completion_tokens": 98, + "reasoning_chars": 221, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1205", + "ok": true, + "error": null, + "completion_tokens": 172, + "reasoning_chars": 335, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-88", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 224, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-509", + "ok": true, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 1036, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-311", + "ok": true, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 292, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-874", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-287", + "ok": true, + "error": null, + "completion_tokens": 266, + "reasoning_chars": 482, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1055", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 297, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1246", + "ok": true, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 210, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-975", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 564, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-280", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 283, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-541", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 592, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-974", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 310, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-827", + "ok": true, + "error": null, + "completion_tokens": 686, + "reasoning_chars": 1717, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-125", + "ok": true, + "error": null, + "completion_tokens": 576, + "reasoning_chars": 1753, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-453", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-942", + "ok": true, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 503, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1199", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-884", + "ok": true, + "error": null, + "completion_tokens": 38, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-855", + "ok": true, + "error": null, + "completion_tokens": 351, + "reasoning_chars": 553, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1270", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 314, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-349", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 287, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1231", + "ok": true, + "error": null, + "completion_tokens": 222, + "reasoning_chars": 572, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1106", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 361, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-803", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 548, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-922", + "ok": false, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-152", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 640, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-920", + "ok": true, + "error": null, + "completion_tokens": 218, + "reasoning_chars": 386, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-207", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-297", + "ok": true, + "error": null, + "completion_tokens": 162, + "reasoning_chars": 332, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-786", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 312, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-101", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 423, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1301", + "ok": true, + "error": null, + "completion_tokens": 296, + "reasoning_chars": 458, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-27", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 423, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1005", + "ok": true, + "error": null, + "completion_tokens": 1317, + "reasoning_chars": 3190, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-659", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 432, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-340", + "ok": true, + "error": null, + "completion_tokens": 5481, + "reasoning_chars": 20299, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-288", + "ok": false, + "error": null, + "completion_tokens": 2728, + "reasoning_chars": 9421, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-763", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-840", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 331, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-742", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 283, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-222", + "ok": true, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 386, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-468", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 765, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-220", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 372, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-662", + "ok": true, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 285, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-269", + "ok": true, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 326, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1193", + "ok": true, + "error": null, + "completion_tokens": 339, + "reasoning_chars": 654, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-270", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 222, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-114", + "ok": true, + "error": null, + "completion_tokens": 355, + "reasoning_chars": 874, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-728", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 465, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-396", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 321, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "math500": { + "task": "math500", + "n": 100, + "hits": 75, + "accuracy_pct": 75.0, + "accuracy_ex_trunc_pct": 75.0, + "truncated": 0, + "completion_tokens_p50": 388, + "completion_tokens_p90": 3754, + "completion_tokens_max": 16866, + "reasoning_chars_p50": 723, + "reasoning_chars_p90": 7721, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1518.5, + "gen_tokens": 129757, + "tok_per_s": 85.5, + "details": [ + { + "id": "math-test/algebra/1553.json", + "ok": true, + "error": null, + "completion_tokens": 449, + "reasoning_chars": 663, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/538.json", + "ok": true, + "error": null, + "completion_tokens": 716, + "reasoning_chars": 923, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/691.json", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 597, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1090.json", + "ok": true, + "error": null, + "completion_tokens": 930, + "reasoning_chars": 1932, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/116.json", + "ok": true, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 235, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1255.json", + "ok": true, + "error": null, + "completion_tokens": 392, + "reasoning_chars": 1150, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1143.json", + "ok": true, + "error": null, + "completion_tokens": 576, + "reasoning_chars": 978, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1201.json", + "ok": true, + "error": null, + "completion_tokens": 3683, + "reasoning_chars": 7538, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/964.json", + "ok": false, + "error": null, + "completion_tokens": 2096, + "reasoning_chars": 3997, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/662.json", + "ok": false, + "error": null, + "completion_tokens": 3308, + "reasoning_chars": 5337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/153.json", + "ok": false, + "error": null, + "completion_tokens": 268, + "reasoning_chars": 332, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/276.json", + "ok": true, + "error": null, + "completion_tokens": 459, + "reasoning_chars": 626, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1840.json", + "ok": true, + "error": null, + "completion_tokens": 374, + "reasoning_chars": 869, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1072.json", + "ok": false, + "error": null, + "completion_tokens": 623, + "reasoning_chars": 916, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1907.json", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 122, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/567.json", + "ok": true, + "error": null, + "completion_tokens": 368, + "reasoning_chars": 462, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1922.json", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 316, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1779.json", + "ok": true, + "error": null, + "completion_tokens": 5901, + "reasoning_chars": 14039, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/951.json", + "ok": true, + "error": null, + "completion_tokens": 288, + "reasoning_chars": 409, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1197.json", + "ok": false, + "error": null, + "completion_tokens": 7367, + "reasoning_chars": 13725, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/853.json", + "ok": true, + "error": null, + "completion_tokens": 1608, + "reasoning_chars": 3153, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/516.json", + "ok": true, + "error": null, + "completion_tokens": 1026, + "reasoning_chars": 1873, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1114.json", + "ok": false, + "error": null, + "completion_tokens": 302, + "reasoning_chars": 852, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1065.json", + "ok": true, + "error": null, + "completion_tokens": 1020, + "reasoning_chars": 1420, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1232.json", + "ok": false, + "error": null, + "completion_tokens": 357, + "reasoning_chars": 339, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/769.json", + "ok": true, + "error": null, + "completion_tokens": 587, + "reasoning_chars": 610, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1791.json", + "ok": false, + "error": null, + "completion_tokens": 3292, + "reasoning_chars": 7721, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/246.json", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 383, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1166.json", + "ok": true, + "error": null, + "completion_tokens": 3754, + "reasoning_chars": 6836, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1410.json", + "ok": false, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 729, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1686.json", + "ok": true, + "error": null, + "completion_tokens": 258, + "reasoning_chars": 406, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/754.json", + "ok": false, + "error": null, + "completion_tokens": 734, + "reasoning_chars": 1427, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/477.json", + "ok": true, + "error": null, + "completion_tokens": 494, + "reasoning_chars": 1731, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2680.json", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 265, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/23957.json", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 245, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1865.json", + "ok": true, + "error": null, + "completion_tokens": 580, + "reasoning_chars": 974, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1014.json", + "ok": true, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 725, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/431.json", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 384, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1425.json", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 238, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2584.json", + "ok": false, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 463, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2470.json", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 296, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/547.json", + "ok": true, + "error": null, + "completion_tokens": 3759, + "reasoning_chars": 10937, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1105.json", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/347.json", + "ok": true, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 355, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1257.json", + "ok": true, + "error": null, + "completion_tokens": 597, + "reasoning_chars": 1647, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1126.json", + "ok": true, + "error": null, + "completion_tokens": 545, + "reasoning_chars": 818, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/239.json", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 288, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1961.json", + "ok": false, + "error": null, + "completion_tokens": 387, + "reasoning_chars": 943, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1388.json", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 271, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1078.json", + "ok": true, + "error": null, + "completion_tokens": 620, + "reasoning_chars": 1068, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2157.json", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 657, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/339.json", + "ok": true, + "error": null, + "completion_tokens": 1756, + "reasoning_chars": 6709, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1761.json", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 384, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1960.json", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 267, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/2086.json", + "ok": true, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 492, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1462.json", + "ok": true, + "error": null, + "completion_tokens": 4880, + "reasoning_chars": 8340, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1060.json", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 424, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1055.json", + "ok": true, + "error": null, + "completion_tokens": 2591, + "reasoning_chars": 7341, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2517.json", + "ok": false, + "error": null, + "completion_tokens": 663, + "reasoning_chars": 1218, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/996.json", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 356, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/893.json", + "ok": true, + "error": null, + "completion_tokens": 577, + "reasoning_chars": 685, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/51.json", + "ok": false, + "error": null, + "completion_tokens": 479, + "reasoning_chars": 1353, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1291.json", + "ok": true, + "error": null, + "completion_tokens": 405, + "reasoning_chars": 574, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/694.json", + "ok": true, + "error": null, + "completion_tokens": 414, + "reasoning_chars": 680, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/711.json", + "ok": false, + "error": null, + "completion_tokens": 16866, + "reasoning_chars": 55768, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1655.json", + "ok": false, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 377, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1936.json", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 245, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1572.json", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 775, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/509.json", + "ok": false, + "error": null, + "completion_tokens": 526, + "reasoning_chars": 723, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/864.json", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 260, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1405.json", + "ok": true, + "error": null, + "completion_tokens": 1372, + "reasoning_chars": 2635, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/192.json", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 191, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2102.json", + "ok": true, + "error": null, + "completion_tokens": 331, + "reasoning_chars": 443, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/843.json", + "ok": true, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 847, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/187.json", + "ok": true, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 699, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/612.json", + "ok": true, + "error": null, + "completion_tokens": 388, + "reasoning_chars": 1518, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/615.json", + "ok": true, + "error": null, + "completion_tokens": 5828, + "reasoning_chars": 16684, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1908.json", + "ok": true, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 390, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/928.json", + "ok": true, + "error": null, + "completion_tokens": 685, + "reasoning_chars": 1344, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/505.json", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 209, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1930.json", + "ok": false, + "error": null, + "completion_tokens": 5717, + "reasoning_chars": 11976, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1934.json", + "ok": true, + "error": null, + "completion_tokens": 349, + "reasoning_chars": 350, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1139.json", + "ok": true, + "error": null, + "completion_tokens": 11238, + "reasoning_chars": 33348, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1000.json", + "ok": true, + "error": null, + "completion_tokens": 629, + "reasoning_chars": 1398, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/183.json", + "ok": true, + "error": null, + "completion_tokens": 812, + "reasoning_chars": 1588, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1275.json", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 665, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2700.json", + "ok": false, + "error": null, + "completion_tokens": 1688, + "reasoning_chars": 4774, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2257.json", + "ok": false, + "error": null, + "completion_tokens": 342, + "reasoning_chars": 483, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/14.json", + "ok": false, + "error": null, + "completion_tokens": 374, + "reasoning_chars": 1175, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1300.json", + "ok": false, + "error": null, + "completion_tokens": 841, + "reasoning_chars": 1301, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/515.json", + "ok": true, + "error": null, + "completion_tokens": 318, + "reasoning_chars": 518, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1423.json", + "ok": false, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 395, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1102.json", + "ok": false, + "error": null, + "completion_tokens": 727, + "reasoning_chars": 1695, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1646.json", + "ok": true, + "error": null, + "completion_tokens": 3937, + "reasoning_chars": 11023, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1272.json", + "ok": false, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 181, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/25149.json", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 439, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/697.json", + "ok": true, + "error": null, + "completion_tokens": 8777, + "reasoning_chars": 17297, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2036.json", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 316, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/2037.json", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 318, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/353.json", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 542, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "bbh": { + "task": "bbh", + "n": 150, + "hits": 113, + "accuracy_pct": 75.33, + "accuracy_ex_trunc_pct": 75.33, + "truncated": 0, + "completion_tokens_p50": 188, + "completion_tokens_p90": 648, + "completion_tokens_max": 7647, + "reasoning_chars_p50": 573, + "reasoning_chars_p90": 1737, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1374.3, + "gen_tokens": 56093, + "tok_per_s": 40.8, + "details": [ + { + "id": "bbh-sports_understanding-101", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 740, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-60", + "ok": true, + "error": null, + "completion_tokens": 383, + "reasoning_chars": 1128, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-113", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-31", + "ok": false, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-144", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-28", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 1030, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-116", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 294, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-67", + "ok": true, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 738, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-3", + "ok": true, + "error": null, + "completion_tokens": 162, + "reasoning_chars": 736, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-29", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-192", + "ok": true, + "error": null, + "completion_tokens": 875, + "reasoning_chars": 1951, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-196", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 773, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-10", + "ok": true, + "error": null, + "completion_tokens": 399, + "reasoning_chars": 696, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-83", + "ok": true, + "error": null, + "completion_tokens": 64, + "reasoning_chars": 267, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-76", + "ok": true, + "error": null, + "completion_tokens": 1908, + "reasoning_chars": 7764, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-212", + "ok": true, + "error": null, + "completion_tokens": 457, + "reasoning_chars": 1354, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-87", + "ok": true, + "error": null, + "completion_tokens": 474, + "reasoning_chars": 1236, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-1", + "ok": true, + "error": null, + "completion_tokens": 519, + "reasoning_chars": 1001, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-140", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 121, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-8", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-138", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 1163, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-221", + "ok": true, + "error": null, + "completion_tokens": 672, + "reasoning_chars": 1737, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-136", + "ok": true, + "error": null, + "completion_tokens": 623, + "reasoning_chars": 1406, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-14", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 62, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-0", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 146, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-207", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-32", + "ok": true, + "error": null, + "completion_tokens": 257, + "reasoning_chars": 1092, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-83", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-220", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-234", + "ok": true, + "error": null, + "completion_tokens": 648, + "reasoning_chars": 1915, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-61", + "ok": false, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-21", + "ok": true, + "error": null, + "completion_tokens": 275, + "reasoning_chars": 662, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-31", + "ok": true, + "error": null, + "completion_tokens": 301, + "reasoning_chars": 1214, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-13", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-211", + "ok": false, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-37", + "ok": true, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 601, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-225", + "ok": true, + "error": null, + "completion_tokens": 1089, + "reasoning_chars": 2624, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-90", + "ok": true, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-101", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-179", + "ok": true, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 63, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-173", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 379, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-105", + "ok": true, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 69, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-88", + "ok": true, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 763, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-68", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-185", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-227", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-129", + "ok": true, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 29, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-238", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-202", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 847, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-93", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 1076, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-93", + "ok": false, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 155, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-161", + "ok": false, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-46", + "ok": true, + "error": null, + "completion_tokens": 93, + "reasoning_chars": 421, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-227", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-158", + "ok": true, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 22, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-205", + "ok": false, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-89", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 164, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-177", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 548, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-65", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-72", + "ok": true, + "error": null, + "completion_tokens": 268, + "reasoning_chars": 495, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-49", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 323, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-54", + "ok": true, + "error": null, + "completion_tokens": 1143, + "reasoning_chars": 3083, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-64", + "ok": false, + "error": null, + "completion_tokens": 538, + "reasoning_chars": 2104, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-86", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-108", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 194, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-194", + "ok": true, + "error": null, + "completion_tokens": 398, + "reasoning_chars": 1130, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-194", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 412, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-75", + "ok": false, + "error": null, + "completion_tokens": 7647, + "reasoning_chars": 27041, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-106", + "ok": true, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 632, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-81", + "ok": true, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 771, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-242", + "ok": true, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-150", + "ok": true, + "error": null, + "completion_tokens": 455, + "reasoning_chars": 1703, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-25", + "ok": true, + "error": null, + "completion_tokens": 406, + "reasoning_chars": 1002, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-139", + "ok": true, + "error": null, + "completion_tokens": 187, + "reasoning_chars": 605, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-12", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 935, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-178", + "ok": true, + "error": null, + "completion_tokens": 506, + "reasoning_chars": 786, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-160", + "ok": true, + "error": null, + "completion_tokens": 499, + "reasoning_chars": 2032, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-115", + "ok": true, + "error": null, + "completion_tokens": 4745, + "reasoning_chars": 17759, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-44", + "ok": true, + "error": null, + "completion_tokens": 397, + "reasoning_chars": 1176, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-189", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 677, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-102", + "ok": true, + "error": null, + "completion_tokens": 508, + "reasoning_chars": 758, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-38", + "ok": true, + "error": null, + "completion_tokens": 366, + "reasoning_chars": 686, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-141", + "ok": true, + "error": null, + "completion_tokens": 399, + "reasoning_chars": 1530, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-38", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 1104, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-168", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-212", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-245", + "ok": true, + "error": null, + "completion_tokens": 228, + "reasoning_chars": 946, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-222", + "ok": true, + "error": null, + "completion_tokens": 536, + "reasoning_chars": 1027, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-244", + "ok": true, + "error": null, + "completion_tokens": 200, + "reasoning_chars": 661, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-35", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 550, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-126", + "ok": false, + "error": null, + "completion_tokens": 859, + "reasoning_chars": 2721, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-90", + "ok": true, + "error": null, + "completion_tokens": 1099, + "reasoning_chars": 3987, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-5", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 597, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-241", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-234", + "ok": true, + "error": null, + "completion_tokens": 632, + "reasoning_chars": 1501, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-233", + "ok": true, + "error": null, + "completion_tokens": 465, + "reasoning_chars": 346, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-5", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-152", + "ok": true, + "error": null, + "completion_tokens": 440, + "reasoning_chars": 1312, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-9", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 125, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-209", + "ok": true, + "error": null, + "completion_tokens": 88, + "reasoning_chars": 143, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-171", + "ok": true, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 949, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-160", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 633, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-166", + "ok": false, + "error": null, + "completion_tokens": 116, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-213", + "ok": true, + "error": null, + "completion_tokens": 1073, + "reasoning_chars": 3564, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-240", + "ok": true, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 77, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-95", + "ok": true, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 76, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-227", + "ok": true, + "error": null, + "completion_tokens": 531, + "reasoning_chars": 1555, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-96", + "ok": false, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 644, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-33", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-4", + "ok": true, + "error": null, + "completion_tokens": 329, + "reasoning_chars": 1327, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-180", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-74", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 1023, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-88", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 459, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-10", + "ok": true, + "error": null, + "completion_tokens": 345, + "reasoning_chars": 642, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-191", + "ok": false, + "error": null, + "completion_tokens": 2955, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-71", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-150", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-188", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 194, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-191", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-203", + "ok": false, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-237", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 824, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-24", + "ok": true, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 99, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-27", + "ok": true, + "error": null, + "completion_tokens": 351, + "reasoning_chars": 1098, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-177", + "ok": true, + "error": null, + "completion_tokens": 349, + "reasoning_chars": 1020, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-192", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-242", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-51", + "ok": false, + "error": null, + "completion_tokens": 603, + "reasoning_chars": 2137, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-96", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-100", + "ok": true, + "error": null, + "completion_tokens": 335, + "reasoning_chars": 1308, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-204", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-58", + "ok": true, + "error": null, + "completion_tokens": 455, + "reasoning_chars": 1430, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-157", + "ok": true, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 2, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-104", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-47", + "ok": true, + "error": null, + "completion_tokens": 422, + "reasoning_chars": 1172, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-150", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 108, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-23", + "ok": true, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 163, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-68", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-174", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 538, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-108", + "ok": true, + "error": null, + "completion_tokens": 899, + "reasoning_chars": 1599, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-203", + "ok": false, + "error": null, + "completion_tokens": 674, + "reasoning_chars": 995, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-115", + "ok": true, + "error": null, + "completion_tokens": 518, + "reasoning_chars": 791, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-223", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 1062, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-221", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-5", + "ok": true, + "error": null, + "completion_tokens": 412, + "reasoning_chars": 1232, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-233", + "ok": true, + "error": null, + "completion_tokens": 1530, + "reasoning_chars": 1580, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-3", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 942, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-119", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 573, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-197", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-103", + "ok": true, + "error": null, + "completion_tokens": 2027, + "reasoning_chars": 8717, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-178", + "ok": false, + "error": null, + "completion_tokens": 445, + "reasoning_chars": 1947, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "humaneval": { + "task": "humaneval", + "n": 164, + "hits": 147, + "accuracy_pct": 89.63, + "accuracy_ex_trunc_pct": 92.45, + "truncated": 5, + "completion_tokens_p50": 583, + "completion_tokens_p90": 3385, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 1500, + "reasoning_chars_p90": 9119, + "reasoning_leaked": null, + "empty": 5, + "length_limited": 5, + "errors": 0, + "elapsed_s": 2231.9, + "gen_tokens": 386060, + "tok_per_s": 173.0, + "details": [ + { + "id": "HumanEval/0", + "ok": false, + "error": null, + "completion_tokens": 433, + "reasoning_chars": 1664, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/1", + "ok": true, + "error": null, + "completion_tokens": 1484, + "reasoning_chars": 5995, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/2", + "ok": true, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 1136, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/3", + "ok": true, + "error": null, + "completion_tokens": 282, + "reasoning_chars": 1080, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/4", + "ok": false, + "error": null, + "completion_tokens": 608, + "reasoning_chars": 2455, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/5", + "ok": true, + "error": null, + "completion_tokens": 347, + "reasoning_chars": 879, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/6", + "ok": true, + "error": null, + "completion_tokens": 907, + "reasoning_chars": 3218, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/7", + "ok": false, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 606, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/8", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 784, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/9", + "ok": true, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 1249, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/10", + "ok": true, + "error": null, + "completion_tokens": 2185, + "reasoning_chars": 7698, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/11", + "ok": true, + "error": null, + "completion_tokens": 1308, + "reasoning_chars": 4734, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/12", + "ok": false, + "error": null, + "completion_tokens": 1349, + "reasoning_chars": 6302, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/13", + "ok": true, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 680, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/14", + "ok": true, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 1644, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/15", + "ok": true, + "error": null, + "completion_tokens": 256, + "reasoning_chars": 710, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/16", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 484, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/17", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 806, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/18", + "ok": true, + "error": null, + "completion_tokens": 542, + "reasoning_chars": 1541, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/19", + "ok": true, + "error": null, + "completion_tokens": 469, + "reasoning_chars": 1402, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/20", + "ok": false, + "error": null, + "completion_tokens": 703, + "reasoning_chars": 2806, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/21", + "ok": false, + "error": null, + "completion_tokens": 1166, + "reasoning_chars": 4922, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/22", + "ok": false, + "error": null, + "completion_tokens": 4587, + "reasoning_chars": 17468, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/23", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 614, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/24", + "ok": true, + "error": null, + "completion_tokens": 1760, + "reasoning_chars": 5825, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/25", + "ok": true, + "error": null, + "completion_tokens": 1471, + "reasoning_chars": 5023, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/26", + "ok": false, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 1500, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/27", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/28", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 417, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/29", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 573, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/30", + "ok": true, + "error": null, + "completion_tokens": 383, + "reasoning_chars": 778, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/31", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 521, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/32", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 109184, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "HumanEval/33", + "ok": true, + "error": null, + "completion_tokens": 637, + "reasoning_chars": 1412, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/34", + "ok": true, + "error": null, + "completion_tokens": 332, + "reasoning_chars": 934, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/35", + "ok": true, + "error": null, + "completion_tokens": 226, + "reasoning_chars": 613, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/36", + "ok": true, + "error": null, + "completion_tokens": 581, + "reasoning_chars": 1127, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/37", + "ok": true, + "error": null, + "completion_tokens": 583, + "reasoning_chars": 1457, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/38", + "ok": false, + "error": null, + "completion_tokens": 359, + "reasoning_chars": 978, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/39", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 59311, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "HumanEval/40", + "ok": true, + "error": null, + "completion_tokens": 830, + "reasoning_chars": 2077, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/41", + "ok": true, + "error": null, + "completion_tokens": 293, + "reasoning_chars": 557, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/42", + "ok": true, + "error": null, + "completion_tokens": 222, + "reasoning_chars": 439, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/43", + "ok": true, + "error": null, + "completion_tokens": 394, + "reasoning_chars": 854, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/44", + "ok": true, + "error": null, + "completion_tokens": 1536, + "reasoning_chars": 4863, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/45", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 197, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/46", + "ok": true, + "error": null, + "completion_tokens": 1442, + "reasoning_chars": 3296, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/47", + "ok": false, + "error": null, + "completion_tokens": 16243, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/48", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 399, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/49", + "ok": true, + "error": null, + "completion_tokens": 405, + "reasoning_chars": 984, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/50", + "ok": false, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 742, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/51", + "ok": true, + "error": null, + "completion_tokens": 348, + "reasoning_chars": 811, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/52", + "ok": true, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 520, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/53", + "ok": true, + "error": null, + "completion_tokens": 187, + "reasoning_chars": 444, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/54", + "ok": true, + "error": null, + "completion_tokens": 454, + "reasoning_chars": 1006, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/55", + "ok": true, + "error": null, + "completion_tokens": 427, + "reasoning_chars": 909, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/56", + "ok": true, + "error": null, + "completion_tokens": 346, + "reasoning_chars": 825, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/57", + "ok": true, + "error": null, + "completion_tokens": 642, + "reasoning_chars": 1795, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/58", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/59", + "ok": true, + "error": null, + "completion_tokens": 633, + "reasoning_chars": 1300, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/60", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 711, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/61", + "ok": true, + "error": null, + "completion_tokens": 254, + "reasoning_chars": 488, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/62", + "ok": true, + "error": null, + "completion_tokens": 310, + "reasoning_chars": 702, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/63", + "ok": true, + "error": null, + "completion_tokens": 606, + "reasoning_chars": 1056, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/64", + "ok": true, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 969, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/65", + "ok": true, + "error": null, + "completion_tokens": 848, + "reasoning_chars": 2745, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/66", + "ok": true, + "error": null, + "completion_tokens": 496, + "reasoning_chars": 1147, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/67", + "ok": true, + "error": null, + "completion_tokens": 464, + "reasoning_chars": 825, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/68", + "ok": true, + "error": null, + "completion_tokens": 791, + "reasoning_chars": 2920, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/69", + "ok": true, + "error": null, + "completion_tokens": 261, + "reasoning_chars": 900, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/70", + "ok": true, + "error": null, + "completion_tokens": 626, + "reasoning_chars": 1690, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/71", + "ok": true, + "error": null, + "completion_tokens": 2760, + "reasoning_chars": 8275, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/72", + "ok": true, + "error": null, + "completion_tokens": 384, + "reasoning_chars": 562, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/73", + "ok": true, + "error": null, + "completion_tokens": 512, + "reasoning_chars": 1028, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/74", + "ok": true, + "error": null, + "completion_tokens": 372, + "reasoning_chars": 1089, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/75", + "ok": true, + "error": null, + "completion_tokens": 3385, + "reasoning_chars": 9938, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/76", + "ok": true, + "error": null, + "completion_tokens": 6444, + "reasoning_chars": 17595, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/77", + "ok": true, + "error": null, + "completion_tokens": 1584, + "reasoning_chars": 4172, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/78", + "ok": true, + "error": null, + "completion_tokens": 320, + "reasoning_chars": 1082, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/79", + "ok": true, + "error": null, + "completion_tokens": 360, + "reasoning_chars": 758, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/80", + "ok": true, + "error": null, + "completion_tokens": 421, + "reasoning_chars": 818, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/81", + "ok": true, + "error": null, + "completion_tokens": 2729, + "reasoning_chars": 6414, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/82", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 662, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/83", + "ok": true, + "error": null, + "completion_tokens": 851, + "reasoning_chars": 2072, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/84", + "ok": true, + "error": null, + "completion_tokens": 317, + "reasoning_chars": 526, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/85", + "ok": true, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 691, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/86", + "ok": true, + "error": null, + "completion_tokens": 699, + "reasoning_chars": 2068, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/87", + "ok": true, + "error": null, + "completion_tokens": 842, + "reasoning_chars": 1853, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/88", + "ok": true, + "error": null, + "completion_tokens": 723, + "reasoning_chars": 1845, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/89", + "ok": true, + "error": null, + "completion_tokens": 557, + "reasoning_chars": 1145, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/90", + "ok": true, + "error": null, + "completion_tokens": 356, + "reasoning_chars": 873, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/91", + "ok": true, + "error": null, + "completion_tokens": 2085, + "reasoning_chars": 7358, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/92", + "ok": true, + "error": null, + "completion_tokens": 2090, + "reasoning_chars": 7632, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/93", + "ok": true, + "error": null, + "completion_tokens": 2077, + "reasoning_chars": 6746, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/94", + "ok": true, + "error": null, + "completion_tokens": 1194, + "reasoning_chars": 2044, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/95", + "ok": true, + "error": null, + "completion_tokens": 1405, + "reasoning_chars": 4604, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/96", + "ok": true, + "error": null, + "completion_tokens": 19790, + "reasoning_chars": 42310, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/97", + "ok": true, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 482, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/98", + "ok": true, + "error": null, + "completion_tokens": 346, + "reasoning_chars": 744, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/99", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 56038, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "HumanEval/100", + "ok": true, + "error": null, + "completion_tokens": 417, + "reasoning_chars": 928, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/101", + "ok": true, + "error": null, + "completion_tokens": 349, + "reasoning_chars": 901, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/102", + "ok": true, + "error": null, + "completion_tokens": 3321, + "reasoning_chars": 8301, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/103", + "ok": true, + "error": null, + "completion_tokens": 1636, + "reasoning_chars": 4656, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/104", + "ok": true, + "error": null, + "completion_tokens": 510, + "reasoning_chars": 1509, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/105", + "ok": true, + "error": null, + "completion_tokens": 1433, + "reasoning_chars": 3513, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/106", + "ok": true, + "error": null, + "completion_tokens": 1691, + "reasoning_chars": 5161, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/107", + "ok": true, + "error": null, + "completion_tokens": 523, + "reasoning_chars": 715, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/108", + "ok": true, + "error": null, + "completion_tokens": 948, + "reasoning_chars": 2382, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/109", + "ok": true, + "error": null, + "completion_tokens": 1273, + "reasoning_chars": 2843, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/110", + "ok": true, + "error": null, + "completion_tokens": 520, + "reasoning_chars": 994, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/111", + "ok": true, + "error": null, + "completion_tokens": 508, + "reasoning_chars": 1140, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/112", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 559, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/113", + "ok": true, + "error": null, + "completion_tokens": 1018, + "reasoning_chars": 2620, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/114", + "ok": true, + "error": null, + "completion_tokens": 1083, + "reasoning_chars": 3126, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/115", + "ok": true, + "error": null, + "completion_tokens": 742, + "reasoning_chars": 1630, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/116", + "ok": true, + "error": null, + "completion_tokens": 6806, + "reasoning_chars": 27334, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/117", + "ok": true, + "error": null, + "completion_tokens": 767, + "reasoning_chars": 1936, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/118", + "ok": true, + "error": null, + "completion_tokens": 1213, + "reasoning_chars": 3876, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/119", + "ok": true, + "error": null, + "completion_tokens": 783, + "reasoning_chars": 2022, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/120", + "ok": true, + "error": null, + "completion_tokens": 583, + "reasoning_chars": 1296, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/121", + "ok": true, + "error": null, + "completion_tokens": 366, + "reasoning_chars": 726, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/122", + "ok": true, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 878, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/123", + "ok": true, + "error": null, + "completion_tokens": 940, + "reasoning_chars": 2196, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/124", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 81483, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "HumanEval/125", + "ok": true, + "error": null, + "completion_tokens": 741, + "reasoning_chars": 2155, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/126", + "ok": true, + "error": null, + "completion_tokens": 935, + "reasoning_chars": 2249, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/127", + "ok": true, + "error": null, + "completion_tokens": 775, + "reasoning_chars": 2381, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/128", + "ok": true, + "error": null, + "completion_tokens": 523, + "reasoning_chars": 1090, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/129", + "ok": true, + "error": null, + "completion_tokens": 11223, + "reasoning_chars": 41060, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/130", + "ok": true, + "error": null, + "completion_tokens": 8813, + "reasoning_chars": 21429, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/131", + "ok": true, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 760, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/132", + "ok": true, + "error": null, + "completion_tokens": 5466, + "reasoning_chars": 20905, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/133", + "ok": true, + "error": null, + "completion_tokens": 477, + "reasoning_chars": 1220, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/134", + "ok": true, + "error": null, + "completion_tokens": 1607, + "reasoning_chars": 5399, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/135", + "ok": true, + "error": null, + "completion_tokens": 400, + "reasoning_chars": 945, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/136", + "ok": true, + "error": null, + "completion_tokens": 528, + "reasoning_chars": 1317, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/137", + "ok": true, + "error": null, + "completion_tokens": 763, + "reasoning_chars": 2070, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/138", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 610, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/139", + "ok": true, + "error": null, + "completion_tokens": 569, + "reasoning_chars": 1611, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/140", + "ok": true, + "error": null, + "completion_tokens": 1472, + "reasoning_chars": 4760, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/141", + "ok": true, + "error": null, + "completion_tokens": 1508, + "reasoning_chars": 4209, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/142", + "ok": true, + "error": null, + "completion_tokens": 1200, + "reasoning_chars": 2726, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/143", + "ok": true, + "error": null, + "completion_tokens": 601, + "reasoning_chars": 1491, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/144", + "ok": true, + "error": null, + "completion_tokens": 629, + "reasoning_chars": 1605, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/145", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 41839, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "HumanEval/146", + "ok": true, + "error": null, + "completion_tokens": 1705, + "reasoning_chars": 4238, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/147", + "ok": true, + "error": null, + "completion_tokens": 4524, + "reasoning_chars": 11416, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/148", + "ok": true, + "error": null, + "completion_tokens": 671, + "reasoning_chars": 1335, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/149", + "ok": true, + "error": null, + "completion_tokens": 497, + "reasoning_chars": 1143, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/150", + "ok": true, + "error": null, + "completion_tokens": 430, + "reasoning_chars": 915, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/151", + "ok": true, + "error": null, + "completion_tokens": 828, + "reasoning_chars": 2544, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/152", + "ok": true, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 483, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/153", + "ok": true, + "error": null, + "completion_tokens": 1258, + "reasoning_chars": 3621, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/154", + "ok": true, + "error": null, + "completion_tokens": 921, + "reasoning_chars": 2497, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/155", + "ok": true, + "error": null, + "completion_tokens": 567, + "reasoning_chars": 1602, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/156", + "ok": true, + "error": null, + "completion_tokens": 805, + "reasoning_chars": 1680, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/157", + "ok": true, + "error": null, + "completion_tokens": 19805, + "reasoning_chars": 64364, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/158", + "ok": false, + "error": null, + "completion_tokens": 612, + "reasoning_chars": 1851, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/159", + "ok": true, + "error": null, + "completion_tokens": 491, + "reasoning_chars": 691, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/160", + "ok": true, + "error": null, + "completion_tokens": 6743, + "reasoning_chars": 26139, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/161", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 644, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/162", + "ok": true, + "error": null, + "completion_tokens": 536, + "reasoning_chars": 1860, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/163", + "ok": true, + "error": null, + "completion_tokens": 3144, + "reasoning_chars": 9119, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "ifeval": { + "task": "ifeval", + "n": 100, + "hits": 76, + "accuracy_pct": 76.0, + "accuracy_ex_trunc_pct": 76.0, + "truncated": 0, + "completion_tokens_p50": 570, + "completion_tokens_p90": 2094, + "completion_tokens_max": 8288, + "reasoning_chars_p50": 1266, + "reasoning_chars_p90": 5954, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1395.4, + "gen_tokens": 98676, + "tok_per_s": 70.7, + "instruction_level_pct": 84.67, + "instructions_checked": 150, + "details": [ + { + "id": "ifeval-153", + "ok": true, + "error": null, + "completion_tokens": 1700, + "reasoning_chars": 4178, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-258", + "ok": true, + "error": null, + "completion_tokens": 212, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-368", + "ok": true, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-85", + "ok": true, + "error": null, + "completion_tokens": 1863, + "reasoning_chars": 5954, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-246", + "ok": true, + "error": null, + "completion_tokens": 1544, + "reasoning_chars": 3592, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-23", + "ok": true, + "error": null, + "completion_tokens": 1565, + "reasoning_chars": 4578, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-372", + "ok": true, + "error": null, + "completion_tokens": 1218, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-387", + "ok": true, + "error": null, + "completion_tokens": 403, + "reasoning_chars": 1174, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-138", + "ok": true, + "error": null, + "completion_tokens": 415, + "reasoning_chars": 206, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-56", + "ok": true, + "error": null, + "completion_tokens": 660, + "reasoning_chars": 2422, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-277", + "ok": true, + "error": null, + "completion_tokens": 680, + "reasoning_chars": 1935, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-407", + "ok": false, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 201, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-171", + "ok": true, + "error": null, + "completion_tokens": 412, + "reasoning_chars": 828, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-96", + "ok": true, + "error": null, + "completion_tokens": 1632, + "reasoning_chars": 1125, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-134", + "ok": true, + "error": null, + "completion_tokens": 3361, + "reasoning_chars": 8432, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-218", + "ok": true, + "error": null, + "completion_tokens": 770, + "reasoning_chars": 1266, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-330", + "ok": false, + "error": null, + "completion_tokens": 1835, + "reasoning_chars": 7191, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-6", + "ok": true, + "error": null, + "completion_tokens": 404, + "reasoning_chars": 1284, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-143", + "ok": true, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 356, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-121", + "ok": true, + "error": null, + "completion_tokens": 436, + "reasoning_chars": 1461, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-219", + "ok": true, + "error": null, + "completion_tokens": 456, + "reasoning_chars": 1597, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-524", + "ok": true, + "error": null, + "completion_tokens": 312, + "reasoning_chars": 345, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-206", + "ok": true, + "error": null, + "completion_tokens": 435, + "reasoning_chars": 1578, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-144", + "ok": true, + "error": null, + "completion_tokens": 705, + "reasoning_chars": 1698, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-447", + "ok": true, + "error": null, + "completion_tokens": 272, + "reasoning_chars": 982, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-492", + "ok": true, + "error": null, + "completion_tokens": 2435, + "reasoning_chars": 9167, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-485", + "ok": true, + "error": null, + "completion_tokens": 1962, + "reasoning_chars": 4193, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-220", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-412", + "ok": true, + "error": null, + "completion_tokens": 762, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-342", + "ok": false, + "error": null, + "completion_tokens": 401, + "reasoning_chars": 422, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-222", + "ok": true, + "error": null, + "completion_tokens": 8288, + "reasoning_chars": 25793, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-131", + "ok": false, + "error": null, + "completion_tokens": 526, + "reasoning_chars": 1745, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-210", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 636, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-137", + "ok": true, + "error": null, + "completion_tokens": 503, + "reasoning_chars": 899, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-464", + "ok": true, + "error": null, + "completion_tokens": 493, + "reasoning_chars": 1801, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-503", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-446", + "ok": true, + "error": null, + "completion_tokens": 823, + "reasoning_chars": 1979, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-2", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-525", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 856, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-27", + "ok": false, + "error": null, + "completion_tokens": 355, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-136", + "ok": true, + "error": null, + "completion_tokens": 954, + "reasoning_chars": 2251, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-286", + "ok": true, + "error": null, + "completion_tokens": 637, + "reasoning_chars": 2352, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-449", + "ok": true, + "error": null, + "completion_tokens": 863, + "reasoning_chars": 3236, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-55", + "ok": true, + "error": null, + "completion_tokens": 1168, + "reasoning_chars": 4271, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-93", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 604, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-399", + "ok": true, + "error": null, + "completion_tokens": 698, + "reasoning_chars": 2318, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-19", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 541, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-229", + "ok": true, + "error": null, + "completion_tokens": 614, + "reasoning_chars": 2078, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-181", + "ok": false, + "error": null, + "completion_tokens": 1014, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-362", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-289", + "ok": true, + "error": null, + "completion_tokens": 825, + "reasoning_chars": 3169, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-356", + "ok": false, + "error": null, + "completion_tokens": 246, + "reasoning_chars": 540, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-115", + "ok": false, + "error": null, + "completion_tokens": 1078, + "reasoning_chars": 4372, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-404", + "ok": false, + "error": null, + "completion_tokens": 768, + "reasoning_chars": 1560, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-243", + "ok": false, + "error": null, + "completion_tokens": 2587, + "reasoning_chars": 7798, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-90", + "ok": true, + "error": null, + "completion_tokens": 764, + "reasoning_chars": 1948, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-54", + "ok": true, + "error": null, + "completion_tokens": 1627, + "reasoning_chars": 5889, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-276", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 308, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-308", + "ok": true, + "error": null, + "completion_tokens": 569, + "reasoning_chars": 1665, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-64", + "ok": false, + "error": null, + "completion_tokens": 2003, + "reasoning_chars": 7878, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-260", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 758, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-337", + "ok": true, + "error": null, + "completion_tokens": 373, + "reasoning_chars": 1050, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-510", + "ok": true, + "error": null, + "completion_tokens": 781, + "reasoning_chars": 823, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-119", + "ok": true, + "error": null, + "completion_tokens": 4265, + "reasoning_chars": 16483, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-5", + "ok": true, + "error": null, + "completion_tokens": 264, + "reasoning_chars": 815, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-468", + "ok": true, + "error": null, + "completion_tokens": 3932, + "reasoning_chars": 10602, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-418", + "ok": true, + "error": null, + "completion_tokens": 505, + "reasoning_chars": 1965, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-476", + "ok": true, + "error": null, + "completion_tokens": 2887, + "reasoning_chars": 5594, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-49", + "ok": true, + "error": null, + "completion_tokens": 5151, + "reasoning_chars": 13955, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-336", + "ok": false, + "error": null, + "completion_tokens": 48, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-422", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 34, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-417", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 603, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-347", + "ok": false, + "error": null, + "completion_tokens": 1661, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-101", + "ok": true, + "error": null, + "completion_tokens": 677, + "reasoning_chars": 518, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-209", + "ok": true, + "error": null, + "completion_tokens": 600, + "reasoning_chars": 1852, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-341", + "ok": true, + "error": null, + "completion_tokens": 1590, + "reasoning_chars": 4513, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-533", + "ok": true, + "error": null, + "completion_tokens": 245, + "reasoning_chars": 625, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-158", + "ok": true, + "error": null, + "completion_tokens": 570, + "reasoning_chars": 1907, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-321", + "ok": true, + "error": null, + "completion_tokens": 664, + "reasoning_chars": 2109, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-311", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-50", + "ok": false, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-518", + "ok": true, + "error": null, + "completion_tokens": 1309, + "reasoning_chars": 1575, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-374", + "ok": true, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 1428, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-317", + "ok": false, + "error": null, + "completion_tokens": 368, + "reasoning_chars": 1343, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-512", + "ok": false, + "error": null, + "completion_tokens": 1096, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-102", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-72", + "ok": false, + "error": null, + "completion_tokens": 785, + "reasoning_chars": 2969, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-33", + "ok": false, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-263", + "ok": true, + "error": null, + "completion_tokens": 426, + "reasoning_chars": 804, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-501", + "ok": true, + "error": null, + "completion_tokens": 1646, + "reasoning_chars": 4962, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-4", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 595, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-444", + "ok": false, + "error": null, + "completion_tokens": 3264, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-228", + "ok": true, + "error": null, + "completion_tokens": 2094, + "reasoning_chars": 260, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-299", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 560, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-278", + "ok": false, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-154", + "ok": true, + "error": null, + "completion_tokens": 744, + "reasoning_chars": 2886, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-301", + "ok": true, + "error": null, + "completion_tokens": 342, + "reasoning_chars": 1044, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-161", + "ok": true, + "error": null, + "completion_tokens": 4248, + "reasoning_chars": 8701, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-394", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-159", + "ok": true, + "error": null, + "completion_tokens": 326, + "reasoning_chars": 1129, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + } + ] + } + } + }, + "ablated-w4a16": { + "started": "2026-10-10 14:32:00", + "finished": "2026-10-10 15:06:32", + "seed": 20260816, + "thinking": true, + "tasks": { + "mmlu": { + "task": "mmlu", + "n": 150, + "hits": 114, + "accuracy_pct": 76.0, + "accuracy_ex_trunc_pct": 77.03, + "truncated": 2, + "completion_tokens_p50": 230, + "completion_tokens_p90": 920, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 652, + "reasoning_chars_p90": 2348, + "reasoning_leaked": null, + "empty": 2, + "length_limited": 2, + "errors": 0, + "elapsed_s": 1341.9, + "gen_tokens": 128545, + "tok_per_s": 95.8, + "details": [ + { + "id": "mmlu-12325", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-154", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 180, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12844", + "ok": true, + "error": null, + "completion_tokens": 476, + "reasoning_chars": 1526, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1153", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 635, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11089", + "ok": true, + "error": null, + "completion_tokens": 475, + "reasoning_chars": 2344, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12782", + "ok": true, + "error": null, + "completion_tokens": 571, + "reasoning_chars": 2960, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5422", + "ok": false, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12363", + "ok": true, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12556", + "ok": false, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 1118, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13163", + "ok": true, + "error": null, + "completion_tokens": 747, + "reasoning_chars": 2344, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-270", + "ok": true, + "error": null, + "completion_tokens": 392, + "reasoning_chars": 980, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6943", + "ok": true, + "error": null, + "completion_tokens": 517, + "reasoning_chars": 2209, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8330", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 609, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9276", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 525, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7584", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 230, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12819", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6163", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 458, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4099", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 774, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10389", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 252, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1539", + "ok": false, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12013", + "ok": true, + "error": null, + "completion_tokens": 768, + "reasoning_chars": 2502, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7814", + "ok": false, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1106", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 82, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1616", + "ok": true, + "error": null, + "completion_tokens": 274, + "reasoning_chars": 538, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-555", + "ok": true, + "error": null, + "completion_tokens": 468, + "reasoning_chars": 1404, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3698", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 885, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-618", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 658, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9842", + "ok": true, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 881, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4229", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1022", + "ok": true, + "error": null, + "completion_tokens": 268, + "reasoning_chars": 989, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6421", + "ok": true, + "error": null, + "completion_tokens": 145, + "reasoning_chars": 706, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1997", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8941", + "ok": false, + "error": null, + "completion_tokens": 368, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-565", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 192, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11364", + "ok": true, + "error": null, + "completion_tokens": 965, + "reasoning_chars": 2269, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5959", + "ok": true, + "error": null, + "completion_tokens": 381, + "reasoning_chars": 1157, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13066", + "ok": false, + "error": null, + "completion_tokens": 364, + "reasoning_chars": 1595, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3122", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 728, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3722", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13839", + "ok": false, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12570", + "ok": true, + "error": null, + "completion_tokens": 623, + "reasoning_chars": 2571, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9959", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10791", + "ok": false, + "error": null, + "completion_tokens": 2073, + "reasoning_chars": 8489, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12268", + "ok": true, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9051", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 650, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12887", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2667", + "ok": true, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 351, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3906", + "ok": true, + "error": null, + "completion_tokens": 416, + "reasoning_chars": 1390, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10304", + "ok": true, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12927", + "ok": false, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6337", + "ok": true, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 136, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3907", + "ok": true, + "error": null, + "completion_tokens": 98, + "reasoning_chars": 144, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12888", + "ok": false, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 870, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10629", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 534, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5801", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 322, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2936", + "ok": true, + "error": null, + "completion_tokens": 596, + "reasoning_chars": 1955, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7389", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9532", + "ok": false, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 565, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8705", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 976, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11776", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 154751, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "mmlu-6836", + "ok": false, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7856", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 605, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9697", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 1693, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10825", + "ok": false, + "error": null, + "completion_tokens": 540, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11178", + "ok": true, + "error": null, + "completion_tokens": 1548, + "reasoning_chars": 6165, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11420", + "ok": true, + "error": null, + "completion_tokens": 934, + "reasoning_chars": 2220, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4658", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 1002, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5403", + "ok": false, + "error": null, + "completion_tokens": 498, + "reasoning_chars": 1757, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6160", + "ok": true, + "error": null, + "completion_tokens": 429, + "reasoning_chars": 1325, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6872", + "ok": false, + "error": null, + "completion_tokens": 5769, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9738", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 735, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3076", + "ok": true, + "error": null, + "completion_tokens": 427, + "reasoning_chars": 1274, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13496", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 866, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11289", + "ok": false, + "error": null, + "completion_tokens": 1210, + "reasoning_chars": 4285, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-222", + "ok": true, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12180", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 776, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1143", + "ok": false, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13086", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 635, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7038", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 668, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13587", + "ok": false, + "error": null, + "completion_tokens": 258, + "reasoning_chars": 588, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12349", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 719, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11747", + "ok": false, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9848", + "ok": false, + "error": null, + "completion_tokens": 484, + "reasoning_chars": 2223, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3438", + "ok": true, + "error": null, + "completion_tokens": 477, + "reasoning_chars": 1307, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-89", + "ok": true, + "error": null, + "completion_tokens": 230, + "reasoning_chars": 928, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4253", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5460", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2494", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 109788, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "mmlu-11149", + "ok": false, + "error": null, + "completion_tokens": 1014, + "reasoning_chars": 3037, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-718", + "ok": true, + "error": null, + "completion_tokens": 433, + "reasoning_chars": 1219, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5330", + "ok": false, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 754, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10716", + "ok": true, + "error": null, + "completion_tokens": 1893, + "reasoning_chars": 6258, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5636", + "ok": false, + "error": null, + "completion_tokens": 736, + "reasoning_chars": 2036, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11832", + "ok": true, + "error": null, + "completion_tokens": 202, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-95", + "ok": true, + "error": null, + "completion_tokens": 443, + "reasoning_chars": 1253, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13960", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 482, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2072", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 17, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12030", + "ok": false, + "error": null, + "completion_tokens": 2162, + "reasoning_chars": 8931, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2931", + "ok": true, + "error": null, + "completion_tokens": 246, + "reasoning_chars": 542, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2441", + "ok": true, + "error": null, + "completion_tokens": 426, + "reasoning_chars": 434, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13666", + "ok": false, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6575", + "ok": true, + "error": null, + "completion_tokens": 137, + "reasoning_chars": 609, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11625", + "ok": true, + "error": null, + "completion_tokens": 920, + "reasoning_chars": 2853, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13283", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4262", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 591, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12931", + "ok": true, + "error": null, + "completion_tokens": 282, + "reasoning_chars": 993, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12150", + "ok": true, + "error": null, + "completion_tokens": 757, + "reasoning_chars": 2296, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3678", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 698, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4593", + "ok": false, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1563", + "ok": true, + "error": null, + "completion_tokens": 526, + "reasoning_chars": 1623, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13396", + "ok": true, + "error": null, + "completion_tokens": 490, + "reasoning_chars": 1390, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8049", + "ok": true, + "error": null, + "completion_tokens": 131, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8396", + "ok": true, + "error": null, + "completion_tokens": 437, + "reasoning_chars": 1245, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8493", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 1017, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13514", + "ok": true, + "error": null, + "completion_tokens": 408, + "reasoning_chars": 1257, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3000", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 152, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1635", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1391", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 571, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-627", + "ok": false, + "error": null, + "completion_tokens": 5847, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1573", + "ok": true, + "error": null, + "completion_tokens": 40, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6731", + "ok": false, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11665", + "ok": true, + "error": null, + "completion_tokens": 1927, + "reasoning_chars": 9394, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2914", + "ok": true, + "error": null, + "completion_tokens": 490, + "reasoning_chars": 1741, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6875", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2184", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12801", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6717", + "ok": true, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12464", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 666, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2787", + "ok": true, + "error": null, + "completion_tokens": 318, + "reasoning_chars": 935, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11549", + "ok": true, + "error": null, + "completion_tokens": 943, + "reasoning_chars": 3454, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-842", + "ok": true, + "error": null, + "completion_tokens": 744, + "reasoning_chars": 2348, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13868", + "ok": true, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9647", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9502", + "ok": true, + "error": null, + "completion_tokens": 617, + "reasoning_chars": 1872, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12441", + "ok": true, + "error": null, + "completion_tokens": 429, + "reasoning_chars": 1316, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4392", + "ok": false, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5036", + "ok": true, + "error": null, + "completion_tokens": 398, + "reasoning_chars": 1395, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12606", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 1577, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1400", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13979", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 617, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13634", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11226", + "ok": true, + "error": null, + "completion_tokens": 1712, + "reasoning_chars": 5962, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-625", + "ok": true, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 1421, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-257", + "ok": true, + "error": null, + "completion_tokens": 569, + "reasoning_chars": 1267, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12853", + "ok": true, + "error": null, + "completion_tokens": 222, + "reasoning_chars": 652, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-451", + "ok": true, + "error": null, + "completion_tokens": 527, + "reasoning_chars": 1580, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2074", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 80, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9117", + "ok": true, + "error": null, + "completion_tokens": 316, + "reasoning_chars": 1304, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10748", + "ok": true, + "error": null, + "completion_tokens": 463, + "reasoning_chars": 1540, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "cmmlu": { + "task": "cmmlu", + "n": 150, + "hits": 113, + "accuracy_pct": 75.33, + "accuracy_ex_trunc_pct": 75.33, + "truncated": 0, + "completion_tokens_p50": 156, + "completion_tokens_p90": 611, + "completion_tokens_max": 3012, + "reasoning_chars_p50": 318, + "reasoning_chars_p90": 1969, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1086.4, + "gen_tokens": 37435, + "tok_per_s": 34.5, + "details": [ + { + "id": "cmmlu-chinese_teacher_qualification-63", + "ok": true, + "error": null, + "completion_tokens": 381, + "reasoning_chars": 1199, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-54", + "ok": true, + "error": null, + "completion_tokens": 695, + "reasoning_chars": 1767, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-91", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-260", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-32", + "ok": true, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 54, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-29", + "ok": true, + "error": null, + "completion_tokens": 402, + "reasoning_chars": 1363, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-200", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-98", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-44", + "ok": true, + "error": null, + "completion_tokens": 166, + "reasoning_chars": 593, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-118", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-116", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-43", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 1008, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-145", + "ok": true, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-247", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 235, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-224", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 171, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-26", + "ok": true, + "error": null, + "completion_tokens": 611, + "reasoning_chars": 1232, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-105", + "ok": false, + "error": null, + "completion_tokens": 1255, + "reasoning_chars": 1606, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-15", + "ok": true, + "error": null, + "completion_tokens": 574, + "reasoning_chars": 1518, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-62", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-nutrition-74", + "ok": true, + "error": null, + "completion_tokens": 296, + "reasoning_chars": 1183, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-181", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 570, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-127", + "ok": true, + "error": null, + "completion_tokens": 612, + "reasoning_chars": 2420, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_geography-107", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-18", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 572, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-82", + "ok": true, + "error": null, + "completion_tokens": 145, + "reasoning_chars": 449, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_food_culture-38", + "ok": true, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 69, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-198", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 115, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-96", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 1031, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-73", + "ok": true, + "error": null, + "completion_tokens": 989, + "reasoning_chars": 3668, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-20", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-95", + "ok": true, + "error": null, + "completion_tokens": 468, + "reasoning_chars": 778, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-88", + "ok": true, + "error": null, + "completion_tokens": 1044, + "reasoning_chars": 4315, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-192", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 361, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-66", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 801, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-100", + "ok": true, + "error": null, + "completion_tokens": 431, + "reasoning_chars": 416, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-136", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 253, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-16", + "ok": false, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 1316, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-90", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-41", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-345", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-103", + "ok": false, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 296, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-93", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-88", + "ok": true, + "error": null, + "completion_tokens": 218, + "reasoning_chars": 379, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-206", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 1010, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-105", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-90", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 1116, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-190", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-international_law-39", + "ok": true, + "error": null, + "completion_tokens": 265, + "reasoning_chars": 914, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-155", + "ok": true, + "error": null, + "completion_tokens": 936, + "reasoning_chars": 2607, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-24", + "ok": true, + "error": null, + "completion_tokens": 301, + "reasoning_chars": 1320, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-6", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-31", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-243", + "ok": true, + "error": null, + "completion_tokens": 688, + "reasoning_chars": 1980, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-63", + "ok": true, + "error": null, + "completion_tokens": 603, + "reasoning_chars": 1980, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-135", + "ok": true, + "error": null, + "completion_tokens": 308, + "reasoning_chars": 1049, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-89", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 164, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-91", + "ok": false, + "error": null, + "completion_tokens": 1192, + "reasoning_chars": 4333, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-120", + "ok": true, + "error": null, + "completion_tokens": 427, + "reasoning_chars": 815, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-62", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-106", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 792, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-food_science-106", + "ok": false, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 318, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-24", + "ok": true, + "error": null, + "completion_tokens": 1204, + "reasoning_chars": 4212, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-157", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-41", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 1524, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-115", + "ok": false, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-journalism-29", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 823, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-67", + "ok": true, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 238, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-161", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 125, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-120", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-education-117", + "ok": true, + "error": null, + "completion_tokens": 156, + "reasoning_chars": 439, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-169", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 930, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-129", + "ok": false, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-19", + "ok": true, + "error": null, + "completion_tokens": 943, + "reasoning_chars": 3275, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-8", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 711, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-72", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 1192, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-public_relations-139", + "ok": false, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 769, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_security-22", + "ok": true, + "error": null, + "completion_tokens": 84, + "reasoning_chars": 392, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-7", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-108", + "ok": true, + "error": null, + "completion_tokens": 697, + "reasoning_chars": 1777, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-31", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-131", + "ok": true, + "error": null, + "completion_tokens": 400, + "reasoning_chars": 653, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-35", + "ok": true, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 51, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_actuarial_science-38", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-165", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-101", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-107", + "ok": true, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 138, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-8", + "ok": false, + "error": null, + "completion_tokens": 3012, + "reasoning_chars": 11890, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medical_statistics-86", + "ok": true, + "error": null, + "completion_tokens": 587, + "reasoning_chars": 2069, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-security_study-30", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 1363, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-118", + "ok": true, + "error": null, + "completion_tokens": 430, + "reasoning_chars": 1697, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_law-24", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-46", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-anatomy-21", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-145", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-arts-91", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 21, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-109", + "ok": false, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 51, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-44", + "ok": true, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-74", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 56, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-63", + "ok": true, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-astronomy-146", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-109", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 789, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-construction_project_management-16", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 161, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-113", + "ok": false, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 978, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-102", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-207", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-19", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-164", + "ok": true, + "error": null, + "completion_tokens": 326, + "reasoning_chars": 1198, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-338", + "ok": true, + "error": null, + "completion_tokens": 40, + "reasoning_chars": 53, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-7", + "ok": true, + "error": null, + "completion_tokens": 478, + "reasoning_chars": 1979, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-92", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-47", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 1511, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-astronomy-140", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 22, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-100", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 961, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-37", + "ok": true, + "error": null, + "completion_tokens": 614, + "reasoning_chars": 1969, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-191", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-34", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 191, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-88", + "ok": true, + "error": null, + "completion_tokens": 421, + "reasoning_chars": 1324, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-public_relations-77", + "ok": false, + "error": null, + "completion_tokens": 602, + "reasoning_chars": 2244, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-306", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-40", + "ok": true, + "error": null, + "completion_tokens": 288, + "reasoning_chars": 1180, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_chemistry-7", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-152", + "ok": true, + "error": null, + "completion_tokens": 475, + "reasoning_chars": 823, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-1", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-94", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 619, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-109", + "ok": true, + "error": null, + "completion_tokens": 192, + "reasoning_chars": 827, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-164", + "ok": true, + "error": null, + "completion_tokens": 116, + "reasoning_chars": 159, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-89", + "ok": false, + "error": null, + "completion_tokens": 290, + "reasoning_chars": 1264, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-17", + "ok": true, + "error": null, + "completion_tokens": 456, + "reasoning_chars": 1475, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-arts-2", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 214, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-51", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-141", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 1065, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-196", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 90, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-97", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 61, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-202", + "ok": false, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-195", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-21", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-49", + "ok": true, + "error": null, + "completion_tokens": 652, + "reasoning_chars": 2573, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-94", + "ok": true, + "error": null, + "completion_tokens": 325, + "reasoning_chars": 1434, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-8", + "ok": false, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 632, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-98", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 277, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-28", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 515, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-74", + "ok": true, + "error": null, + "completion_tokens": 839, + "reasoning_chars": 2880, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-121", + "ok": true, + "error": null, + "completion_tokens": 406, + "reasoning_chars": 1204, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-120", + "ok": true, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 481, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-105", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 152, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-35", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 677, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-51", + "ok": true, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 897, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-49", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-130", + "ok": true, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 1441, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-140", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "mmlu_pro": { + "task": "mmlu_pro", + "n": 150, + "hits": 106, + "accuracy_pct": 70.67, + "accuracy_ex_trunc_pct": 71.14, + "truncated": 1, + "completion_tokens_p50": 374, + "completion_tokens_p90": 2631, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 678, + "reasoning_chars_p90": 3617, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 949.4, + "gen_tokens": 241357, + "tok_per_s": 254.2, + "details": [ + { + "id": "mmlupro-5831", + "ok": true, + "error": null, + "completion_tokens": 269, + "reasoning_chars": 721, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2299", + "ok": true, + "error": null, + "completion_tokens": 590, + "reasoning_chars": 2136, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9280", + "ok": false, + "error": null, + "completion_tokens": 200, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5486", + "ok": true, + "error": null, + "completion_tokens": 762, + "reasoning_chars": 3617, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5761", + "ok": false, + "error": null, + "completion_tokens": 116, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6589", + "ok": true, + "error": null, + "completion_tokens": 734, + "reasoning_chars": 1904, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5389", + "ok": true, + "error": null, + "completion_tokens": 404, + "reasoning_chars": 1336, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5131", + "ok": true, + "error": null, + "completion_tokens": 838, + "reasoning_chars": 2111, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1677", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4388", + "ok": false, + "error": null, + "completion_tokens": 28753, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3285", + "ok": true, + "error": null, + "completion_tokens": 935, + "reasoning_chars": 2392, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4118", + "ok": true, + "error": null, + "completion_tokens": 1434, + "reasoning_chars": 3699, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5344", + "ok": false, + "error": null, + "completion_tokens": 64, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11696", + "ok": false, + "error": null, + "completion_tokens": 4133, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-858", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-214", + "ok": false, + "error": null, + "completion_tokens": 4667, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6609", + "ok": false, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2083", + "ok": true, + "error": null, + "completion_tokens": 772, + "reasoning_chars": 2871, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9752", + "ok": false, + "error": null, + "completion_tokens": 10261, + "reasoning_chars": 22761, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7213", + "ok": true, + "error": null, + "completion_tokens": 500, + "reasoning_chars": 1623, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5234", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 594, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8570", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5612", + "ok": false, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2071", + "ok": true, + "error": null, + "completion_tokens": 692, + "reasoning_chars": 2485, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9842", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4665", + "ok": true, + "error": null, + "completion_tokens": 957, + "reasoning_chars": 2120, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5564", + "ok": true, + "error": null, + "completion_tokens": 466, + "reasoning_chars": 1974, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6475", + "ok": true, + "error": null, + "completion_tokens": 910, + "reasoning_chars": 2549, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2153", + "ok": true, + "error": null, + "completion_tokens": 246, + "reasoning_chars": 930, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5812", + "ok": true, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 247, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3407", + "ok": true, + "error": null, + "completion_tokens": 2631, + "reasoning_chars": 4215, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2879", + "ok": false, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 495, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8266", + "ok": false, + "error": null, + "completion_tokens": 54, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1916", + "ok": true, + "error": null, + "completion_tokens": 822, + "reasoning_chars": 3127, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3486", + "ok": true, + "error": null, + "completion_tokens": 534, + "reasoning_chars": 1943, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11275", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6040", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8565", + "ok": true, + "error": null, + "completion_tokens": 162, + "reasoning_chars": 249, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6848", + "ok": true, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 303, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8926", + "ok": false, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8458", + "ok": true, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 98, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7854", + "ok": true, + "error": null, + "completion_tokens": 388, + "reasoning_chars": 1155, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3962", + "ok": true, + "error": null, + "completion_tokens": 7209, + "reasoning_chars": 8947, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6470", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3813", + "ok": true, + "error": null, + "completion_tokens": 376, + "reasoning_chars": 1012, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8026", + "ok": true, + "error": null, + "completion_tokens": 54, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11030", + "ok": true, + "error": null, + "completion_tokens": 693, + "reasoning_chars": 2203, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8151", + "ok": true, + "error": null, + "completion_tokens": 40, + "reasoning_chars": 15, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5286", + "ok": true, + "error": null, + "completion_tokens": 258, + "reasoning_chars": 890, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3888", + "ok": true, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11676", + "ok": false, + "error": null, + "completion_tokens": 1016, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8762", + "ok": false, + "error": null, + "completion_tokens": 794, + "reasoning_chars": 1515, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-12160", + "ok": false, + "error": null, + "completion_tokens": 539, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4022", + "ok": true, + "error": null, + "completion_tokens": 797, + "reasoning_chars": 2290, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10092", + "ok": true, + "error": null, + "completion_tokens": 2959, + "reasoning_chars": 9541, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7364", + "ok": true, + "error": null, + "completion_tokens": 277, + "reasoning_chars": 701, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6001", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10873", + "ok": true, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9777", + "ok": true, + "error": null, + "completion_tokens": 2957, + "reasoning_chars": 4812, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8723", + "ok": true, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4595", + "ok": true, + "error": null, + "completion_tokens": 374, + "reasoning_chars": 921, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8423", + "ok": false, + "error": null, + "completion_tokens": 1490, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11051", + "ok": false, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 678, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-836", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11297", + "ok": true, + "error": null, + "completion_tokens": 31236, + "reasoning_chars": 85173, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3950", + "ok": true, + "error": null, + "completion_tokens": 296, + "reasoning_chars": 905, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1562", + "ok": false, + "error": null, + "completion_tokens": 552, + "reasoning_chars": 1952, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2041", + "ok": true, + "error": null, + "completion_tokens": 744, + "reasoning_chars": 2725, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3473", + "ok": true, + "error": null, + "completion_tokens": 54, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11253", + "ok": false, + "error": null, + "completion_tokens": 276, + "reasoning_chars": 1283, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3858", + "ok": true, + "error": null, + "completion_tokens": 716, + "reasoning_chars": 1262, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2231", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2190", + "ok": false, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6930", + "ok": true, + "error": null, + "completion_tokens": 489, + "reasoning_chars": 1992, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6558", + "ok": true, + "error": null, + "completion_tokens": 782, + "reasoning_chars": 2394, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5371", + "ok": true, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1766", + "ok": true, + "error": null, + "completion_tokens": 898, + "reasoning_chars": 2986, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6045", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2298", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 882, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6400", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6499", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7936", + "ok": true, + "error": null, + "completion_tokens": 289, + "reasoning_chars": 1146, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8566", + "ok": true, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11633", + "ok": true, + "error": null, + "completion_tokens": 685, + "reasoning_chars": 783, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9843", + "ok": false, + "error": null, + "completion_tokens": 4328, + "reasoning_chars": 7266, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11758", + "ok": false, + "error": null, + "completion_tokens": 24725, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2843", + "ok": true, + "error": null, + "completion_tokens": 1551, + "reasoning_chars": 4524, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7483", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11379", + "ok": true, + "error": null, + "completion_tokens": 3403, + "reasoning_chars": 5489, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4866", + "ok": true, + "error": null, + "completion_tokens": 701, + "reasoning_chars": 2188, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-818", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 155, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2785", + "ok": true, + "error": null, + "completion_tokens": 845, + "reasoning_chars": 3191, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8119", + "ok": false, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2214", + "ok": true, + "error": null, + "completion_tokens": 594, + "reasoning_chars": 1534, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5774", + "ok": true, + "error": null, + "completion_tokens": 1317, + "reasoning_chars": 4871, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6049", + "ok": false, + "error": null, + "completion_tokens": 473, + "reasoning_chars": 1485, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7219", + "ok": true, + "error": null, + "completion_tokens": 424, + "reasoning_chars": 1399, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1660", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4767", + "ok": true, + "error": null, + "completion_tokens": 62, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5964", + "ok": false, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 1095, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2743", + "ok": true, + "error": null, + "completion_tokens": 45, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4062", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 148, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7368", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-827", + "ok": false, + "error": null, + "completion_tokens": 569, + "reasoning_chars": 2414, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11877", + "ok": false, + "error": null, + "completion_tokens": 8798, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7557", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9998", + "ok": false, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10098", + "ok": false, + "error": null, + "completion_tokens": 935, + "reasoning_chars": 2838, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1744", + "ok": false, + "error": null, + "completion_tokens": 1053, + "reasoning_chars": 3786, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9210", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 509, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11248", + "ok": false, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11187", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2799", + "ok": true, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4479", + "ok": true, + "error": null, + "completion_tokens": 1396, + "reasoning_chars": 2703, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10223", + "ok": true, + "error": null, + "completion_tokens": 1292, + "reasoning_chars": 1644, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5698", + "ok": true, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 411, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8291", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 140, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3516", + "ok": true, + "error": null, + "completion_tokens": 781, + "reasoning_chars": 2016, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5466", + "ok": true, + "error": null, + "completion_tokens": 916, + "reasoning_chars": 2937, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4851", + "ok": true, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-733", + "ok": true, + "error": null, + "completion_tokens": 1880, + "reasoning_chars": 2793, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6995", + "ok": true, + "error": null, + "completion_tokens": 423, + "reasoning_chars": 1231, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2573", + "ok": true, + "error": null, + "completion_tokens": 649, + "reasoning_chars": 2767, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8481", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 438, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5938", + "ok": true, + "error": null, + "completion_tokens": 741, + "reasoning_chars": 2781, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10708", + "ok": false, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8476", + "ok": true, + "error": null, + "completion_tokens": 445, + "reasoning_chars": 1244, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7990", + "ok": true, + "error": null, + "completion_tokens": 302, + "reasoning_chars": 961, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2409", + "ok": true, + "error": null, + "completion_tokens": 576, + "reasoning_chars": 2235, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11551", + "ok": true, + "error": null, + "completion_tokens": 547, + "reasoning_chars": 454, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-674", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 132, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1978", + "ok": true, + "error": null, + "completion_tokens": 1969, + "reasoning_chars": 7713, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10420", + "ok": false, + "error": null, + "completion_tokens": 137, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3445", + "ok": true, + "error": null, + "completion_tokens": 981, + "reasoning_chars": 3429, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9488", + "ok": true, + "error": null, + "completion_tokens": 655, + "reasoning_chars": 657, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4993", + "ok": true, + "error": null, + "completion_tokens": 397, + "reasoning_chars": 1364, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4316", + "ok": true, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4730", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 807, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11570", + "ok": false, + "error": null, + "completion_tokens": 48, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2149", + "ok": true, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 1316, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8322", + "ok": true, + "error": null, + "completion_tokens": 8483, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10993", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4588", + "ok": false, + "error": null, + "completion_tokens": 7429, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3258", + "ok": true, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8629", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 415, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-637", + "ok": true, + "error": null, + "completion_tokens": 319, + "reasoning_chars": 359, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4257", + "ok": true, + "error": null, + "completion_tokens": 317, + "reasoning_chars": 799, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9719", + "ok": true, + "error": null, + "completion_tokens": 701, + "reasoning_chars": 1810, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11714", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 74659, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "mmlupro-9678", + "ok": false, + "error": null, + "completion_tokens": 1355, + "reasoning_chars": 4710, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "ceval": { + "task": "ceval", + "n": 150, + "hits": 114, + "accuracy_pct": 76.0, + "accuracy_ex_trunc_pct": 76.51, + "truncated": 1, + "completion_tokens_p50": 197, + "completion_tokens_p90": 1003, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 403, + "reasoning_chars_p90": 2994, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 1347.6, + "gen_tokens": 108193, + "tok_per_s": 80.3, + "details": [ + { + "id": "ceval-high_school_chinese-1", + "ok": false, + "error": null, + "completion_tokens": 918, + "reasoning_chars": 1439, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-10", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-41", + "ok": false, + "error": null, + "completion_tokens": 3944, + "reasoning_chars": 10623, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_physics-3", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-21", + "ok": true, + "error": null, + "completion_tokens": 645, + "reasoning_chars": 2994, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-4", + "ok": true, + "error": null, + "completion_tokens": 2964, + "reasoning_chars": 6989, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-21", + "ok": true, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 1029, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-2", + "ok": true, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 142, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-6", + "ok": true, + "error": null, + "completion_tokens": 98, + "reasoning_chars": 278, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-12", + "ok": false, + "error": null, + "completion_tokens": 1598, + "reasoning_chars": 5557, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-19", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_chemistry-9", + "ok": true, + "error": null, + "completion_tokens": 3947, + "reasoning_chars": 14603, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-15", + "ok": true, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-22", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-16", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 586, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-11", + "ok": false, + "error": null, + "completion_tokens": 218, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-18", + "ok": true, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 158, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-24", + "ok": true, + "error": null, + "completion_tokens": 888, + "reasoning_chars": 2759, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-29", + "ok": true, + "error": null, + "completion_tokens": 1573, + "reasoning_chars": 5949, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-2", + "ok": true, + "error": null, + "completion_tokens": 356, + "reasoning_chars": 504, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-11", + "ok": false, + "error": null, + "completion_tokens": 245, + "reasoning_chars": 1109, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-18", + "ok": true, + "error": null, + "completion_tokens": 465, + "reasoning_chars": 1808, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-0", + "ok": false, + "error": null, + "completion_tokens": 379, + "reasoning_chars": 1397, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-14", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-27", + "ok": true, + "error": null, + "completion_tokens": 659, + "reasoning_chars": 2771, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-art_studies-19", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 277, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-21", + "ok": true, + "error": null, + "completion_tokens": 1072, + "reasoning_chars": 3558, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-33", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 403, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-20", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-10", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-19", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_network-7", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 59, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-41", + "ok": true, + "error": null, + "completion_tokens": 439, + "reasoning_chars": 1301, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-44", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 1254, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-9", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 468, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-28", + "ok": true, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 1369, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-39", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-1", + "ok": false, + "error": null, + "completion_tokens": 301, + "reasoning_chars": 992, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-24", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 612, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-5", + "ok": true, + "error": null, + "completion_tokens": 1616, + "reasoning_chars": 5596, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_architecture-15", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 169, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-39", + "ok": true, + "error": null, + "completion_tokens": 409, + "reasoning_chars": 1914, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-23", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 323, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-7", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 1123, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-17", + "ok": true, + "error": null, + "completion_tokens": 224, + "reasoning_chars": 1056, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_chemistry-6", + "ok": true, + "error": null, + "completion_tokens": 386, + "reasoning_chars": 850, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-15", + "ok": true, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 112, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-4", + "ok": false, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-5", + "ok": true, + "error": null, + "completion_tokens": 811, + "reasoning_chars": 3293, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-50", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_politics-11", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-32", + "ok": true, + "error": null, + "completion_tokens": 822, + "reasoning_chars": 2600, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-16", + "ok": true, + "error": null, + "completion_tokens": 873, + "reasoning_chars": 2367, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-0", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-sports_science-9", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 293, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-art_studies-15", + "ok": true, + "error": null, + "completion_tokens": 95, + "reasoning_chars": 109, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-2", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-7", + "ok": true, + "error": null, + "completion_tokens": 865, + "reasoning_chars": 2931, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-11", + "ok": false, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-operating_system-18", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 139, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-4", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-17", + "ok": true, + "error": null, + "completion_tokens": 466, + "reasoning_chars": 1245, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-1", + "ok": true, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-32", + "ok": true, + "error": null, + "completion_tokens": 172, + "reasoning_chars": 147, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-33", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-25", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 897, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-40", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-7", + "ok": true, + "error": null, + "completion_tokens": 1176, + "reasoning_chars": 4286, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-5", + "ok": true, + "error": null, + "completion_tokens": 367, + "reasoning_chars": 1742, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-12", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-4", + "ok": true, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 235, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-38", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 44005, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "ceval-electrical_engineer-22", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 693, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-0", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-15", + "ok": false, + "error": null, + "completion_tokens": 524, + "reasoning_chars": 1723, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-45", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-6", + "ok": true, + "error": null, + "completion_tokens": 609, + "reasoning_chars": 889, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-20", + "ok": true, + "error": null, + "completion_tokens": 981, + "reasoning_chars": 3145, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-basic_medicine-2", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-17", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-16", + "ok": false, + "error": null, + "completion_tokens": 509, + "reasoning_chars": 2418, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-7", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 784, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_biology-16", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-10", + "ok": false, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-32", + "ok": true, + "error": null, + "completion_tokens": 7817, + "reasoning_chars": 21220, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-0", + "ok": true, + "error": null, + "completion_tokens": 95, + "reasoning_chars": 134, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-12", + "ok": true, + "error": null, + "completion_tokens": 981, + "reasoning_chars": 1710, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-14", + "ok": true, + "error": null, + "completion_tokens": 654, + "reasoning_chars": 2626, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-26", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-3", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-3", + "ok": true, + "error": null, + "completion_tokens": 101, + "reasoning_chars": 128, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_network-13", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 338, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-sports_science-5", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-27", + "ok": true, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 705, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-21", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-10", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 233, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-16", + "ok": false, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 36, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-49", + "ok": true, + "error": null, + "completion_tokens": 302, + "reasoning_chars": 1219, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-13", + "ok": false, + "error": null, + "completion_tokens": 648, + "reasoning_chars": 2015, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-5", + "ok": true, + "error": null, + "completion_tokens": 4266, + "reasoning_chars": 11569, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-2", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 925, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-27", + "ok": false, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 1039, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_biology-8", + "ok": false, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_physics-14", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-14", + "ok": true, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 508, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-31", + "ok": true, + "error": null, + "completion_tokens": 448, + "reasoning_chars": 724, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-basic_medicine-13", + "ok": true, + "error": null, + "completion_tokens": 384, + "reasoning_chars": 1332, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-13", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-12", + "ok": true, + "error": null, + "completion_tokens": 427, + "reasoning_chars": 1129, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-44", + "ok": false, + "error": null, + "completion_tokens": 1172, + "reasoning_chars": 1807, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-3", + "ok": true, + "error": null, + "completion_tokens": 1576, + "reasoning_chars": 4636, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-13", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_mathematics-6", + "ok": true, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 78, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_politics-19", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 919, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-environmental_impact_assessment_engineer-27", + "ok": false, + "error": null, + "completion_tokens": 9119, + "reasoning_chars": 22990, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-10", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 334, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-0", + "ok": true, + "error": null, + "completion_tokens": 381, + "reasoning_chars": 1555, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-3", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 192, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-6", + "ok": true, + "error": null, + "completion_tokens": 400, + "reasoning_chars": 1903, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_chinese-13", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-43", + "ok": false, + "error": null, + "completion_tokens": 425, + "reasoning_chars": 1428, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-42", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 172, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-13", + "ok": true, + "error": null, + "completion_tokens": 406, + "reasoning_chars": 878, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-discrete_mathematics-1", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 89, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-11", + "ok": false, + "error": null, + "completion_tokens": 83, + "reasoning_chars": 360, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 1003, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-2", + "ok": true, + "error": null, + "completion_tokens": 309, + "reasoning_chars": 777, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-1", + "ok": true, + "error": null, + "completion_tokens": 350, + "reasoning_chars": 843, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_physics-16", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-11", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 967, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-5", + "ok": true, + "error": null, + "completion_tokens": 272, + "reasoning_chars": 1132, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-40", + "ok": true, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 1754, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-6", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_mathematics-10", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 7, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-10", + "ok": true, + "error": null, + "completion_tokens": 1399, + "reasoning_chars": 2522, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-17", + "ok": false, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 238, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-11", + "ok": true, + "error": null, + "completion_tokens": 512, + "reasoning_chars": 1794, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-16", + "ok": true, + "error": null, + "completion_tokens": 83, + "reasoning_chars": 121, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-1", + "ok": true, + "error": null, + "completion_tokens": 551, + "reasoning_chars": 1473, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_physics-0", + "ok": false, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-37", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 1105, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-9", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 35, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 178, + "reasoning_chars": 254, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-5", + "ok": true, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 23, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-18", + "ok": true, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 116, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-30", + "ok": true, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 334, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-21", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 790, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-42", + "ok": true, + "error": null, + "completion_tokens": 798, + "reasoning_chars": 1300, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-5", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_chemistry-4", + "ok": true, + "error": null, + "completion_tokens": 286, + "reasoning_chars": 825, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "arc": { + "task": "arc", + "n": 150, + "hits": 128, + "accuracy_pct": 85.33, + "accuracy_ex_trunc_pct": 85.33, + "truncated": 0, + "completion_tokens_p50": 244, + "completion_tokens_p90": 446, + "completion_tokens_max": 1431, + "reasoning_chars_p50": 730, + "reasoning_chars_p90": 1388, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1331.1, + "gen_tokens": 38806, + "tok_per_s": 29.2, + "details": [ + { + "id": "arc-Mercury_415084", + "ok": true, + "error": null, + "completion_tokens": 48, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7249708", + "ok": true, + "error": null, + "completion_tokens": 442, + "reasoning_chars": 1325, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_4_24", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 858, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7056823", + "ok": true, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 710, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415534", + "ok": true, + "error": null, + "completion_tokens": 178, + "reasoning_chars": 419, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405827", + "ok": true, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 1053, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7263183", + "ok": true, + "error": null, + "completion_tokens": 202, + "reasoning_chars": 559, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7106960", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 195, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_7", + "ok": true, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7182245", + "ok": true, + "error": null, + "completion_tokens": 453, + "reasoning_chars": 1611, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416177", + "ok": false, + "error": null, + "completion_tokens": 101, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-ACTAAP_2011_5_8", + "ok": true, + "error": null, + "completion_tokens": 387, + "reasoning_chars": 1095, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_8_36", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 127, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401699", + "ok": true, + "error": null, + "completion_tokens": 101, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7107240", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 839, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_4_26", + "ok": true, + "error": null, + "completion_tokens": 329, + "reasoning_chars": 791, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402638", + "ok": true, + "error": null, + "completion_tokens": 391, + "reasoning_chars": 1090, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7026723", + "ok": true, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 155, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415422", + "ok": true, + "error": null, + "completion_tokens": 402, + "reasoning_chars": 959, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2000_8_2", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 821, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2006_9_4", + "ok": false, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401272", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 392, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416169", + "ok": true, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 97, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7116235", + "ok": true, + "error": null, + "completion_tokens": 467, + "reasoning_chars": 1394, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7109463", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 717, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7032428", + "ok": false, + "error": null, + "completion_tokens": 593, + "reasoning_chars": 1906, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AIMS_2008_8_11", + "ok": true, + "error": null, + "completion_tokens": 386, + "reasoning_chars": 1126, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_410334", + "ok": true, + "error": null, + "completion_tokens": 317, + "reasoning_chars": 1008, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_36", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 401, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415719", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_17", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 700, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402256", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 670, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_400443", + "ok": true, + "error": null, + "completion_tokens": 217, + "reasoning_chars": 311, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7133245", + "ok": true, + "error": null, + "completion_tokens": 336, + "reasoning_chars": 1171, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413458", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401372", + "ok": true, + "error": null, + "completion_tokens": 259, + "reasoning_chars": 884, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401116", + "ok": false, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP__5_10312", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 416, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2007_4_10280", + "ok": true, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 188, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_416683", + "ok": true, + "error": null, + "completion_tokens": 543, + "reasoning_chars": 1672, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7190120", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 571, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401295", + "ok": true, + "error": null, + "completion_tokens": 373, + "reasoning_chars": 1439, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_4_28", + "ok": true, + "error": null, + "completion_tokens": 69, + "reasoning_chars": 287, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7223493", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 877, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7018410", + "ok": false, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1998_8_24", + "ok": false, + "error": null, + "completion_tokens": 325, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_31", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 686, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405501", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 878, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408030", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401684", + "ok": true, + "error": null, + "completion_tokens": 317, + "reasoning_chars": 763, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7041668", + "ok": true, + "error": null, + "completion_tokens": 290, + "reasoning_chars": 850, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7205870", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 909, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7210140", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 1108, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_405873", + "ok": true, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 1431, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400052", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 683, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7018270", + "ok": true, + "error": null, + "completion_tokens": 390, + "reasoning_chars": 1423, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_406550", + "ok": true, + "error": null, + "completion_tokens": 347, + "reasoning_chars": 844, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7150728", + "ok": true, + "error": null, + "completion_tokens": 452, + "reasoning_chars": 1276, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400839", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 859, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_30", + "ok": true, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 298, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400383", + "ok": true, + "error": null, + "completion_tokens": 500, + "reasoning_chars": 1351, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7236740", + "ok": true, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 263, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-CSZ_2005_5_CSZ10021", + "ok": false, + "error": null, + "completion_tokens": 217, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7093958", + "ok": true, + "error": null, + "completion_tokens": 264, + "reasoning_chars": 867, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7026355", + "ok": true, + "error": null, + "completion_tokens": 485, + "reasoning_chars": 1448, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408324", + "ok": true, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 87, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP__7_10348", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 413, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402629", + "ok": true, + "error": null, + "completion_tokens": 93, + "reasoning_chars": 193, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400046", + "ok": true, + "error": null, + "completion_tokens": 329, + "reasoning_chars": 1018, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7180810", + "ok": true, + "error": null, + "completion_tokens": 554, + "reasoning_chars": 1748, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_180005", + "ok": false, + "error": null, + "completion_tokens": 86, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AIMS_2009_8_14", + "ok": true, + "error": null, + "completion_tokens": 316, + "reasoning_chars": 1271, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2007_5_16", + "ok": true, + "error": null, + "completion_tokens": 101, + "reasoning_chars": 419, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_LBS10662", + "ok": true, + "error": null, + "completion_tokens": 316, + "reasoning_chars": 987, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_23", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 703, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2012_8_19", + "ok": false, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 504, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7041650", + "ok": true, + "error": null, + "completion_tokens": 350, + "reasoning_chars": 887, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416181", + "ok": true, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 288, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TIMSS_1995_8_J6", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 124, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2014_5_13", + "ok": true, + "error": null, + "completion_tokens": 332, + "reasoning_chars": 1015, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7106750", + "ok": true, + "error": null, + "completion_tokens": 392, + "reasoning_chars": 973, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1999_4_23", + "ok": true, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 552, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MEA_2013_5_12", + "ok": true, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 150, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_8_42", + "ok": false, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 342, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408367", + "ok": false, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7250058", + "ok": true, + "error": null, + "completion_tokens": 399, + "reasoning_chars": 874, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_414041", + "ok": true, + "error": null, + "completion_tokens": 441, + "reasoning_chars": 1139, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401290", + "ok": false, + "error": null, + "completion_tokens": 246, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_409295", + "ok": true, + "error": null, + "completion_tokens": 577, + "reasoning_chars": 1651, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7211680", + "ok": true, + "error": null, + "completion_tokens": 467, + "reasoning_chars": 1388, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_37", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 1067, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413439", + "ok": true, + "error": null, + "completion_tokens": 428, + "reasoning_chars": 1211, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_32", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 805, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7216423", + "ok": true, + "error": null, + "completion_tokens": 343, + "reasoning_chars": 1047, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-ACTAAP_2015_7_9", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 203, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7230405", + "ok": true, + "error": null, + "completion_tokens": 573, + "reasoning_chars": 1556, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7025008", + "ok": true, + "error": null, + "completion_tokens": 427, + "reasoning_chars": 1044, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_416536", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 426, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TIMSS_2007_8_pg60", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 492, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2009_4_34", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 881, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_406833", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 265, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_409647", + "ok": true, + "error": null, + "completion_tokens": 494, + "reasoning_chars": 1389, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_184363", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 754, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7107170", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 662, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416104", + "ok": false, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_10-v1", + "ok": true, + "error": null, + "completion_tokens": 69, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7202843", + "ok": true, + "error": null, + "completion_tokens": 472, + "reasoning_chars": 1338, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7029803", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 1132, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7164623", + "ok": true, + "error": null, + "completion_tokens": 290, + "reasoning_chars": 1064, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_26", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 442, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7263060", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 1183, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7268328", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 884, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402282", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 670, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7029558", + "ok": true, + "error": null, + "completion_tokens": 54, + "reasoning_chars": 174, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_4_21", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7186358", + "ok": true, + "error": null, + "completion_tokens": 355, + "reasoning_chars": 1344, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2011_8_10434", + "ok": true, + "error": null, + "completion_tokens": 427, + "reasoning_chars": 1191, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2012_8_23651", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 851, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_407125", + "ok": true, + "error": null, + "completion_tokens": 593, + "reasoning_chars": 2020, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7268258", + "ok": false, + "error": null, + "completion_tokens": 56, + "reasoning_chars": 244, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7027405", + "ok": true, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 826, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_8_7", + "ok": true, + "error": null, + "completion_tokens": 1431, + "reasoning_chars": 6785, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_191625", + "ok": false, + "error": null, + "completion_tokens": 269, + "reasoning_chars": 1057, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_6", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 237, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7219678", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 1188, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401678", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 766, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401312", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 376, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_8_42", + "ok": false, + "error": null, + "completion_tokens": 203, + "reasoning_chars": 149, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7030468", + "ok": true, + "error": null, + "completion_tokens": 424, + "reasoning_chars": 1136, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_175385", + "ok": true, + "error": null, + "completion_tokens": 419, + "reasoning_chars": 911, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_175560", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_5", + "ok": true, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_409009", + "ok": true, + "error": null, + "completion_tokens": 391, + "reasoning_chars": 929, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7128923", + "ok": false, + "error": null, + "completion_tokens": 319, + "reasoning_chars": 753, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_8_9", + "ok": false, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 642, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7042945", + "ok": false, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_8_47", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 243, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7185343", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7218400", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 534, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7221988", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 522, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_405950", + "ok": true, + "error": null, + "completion_tokens": 391, + "reasoning_chars": 1156, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7222530", + "ok": true, + "error": null, + "completion_tokens": 417, + "reasoning_chars": 1246, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7102323", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2009_5_39", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 589, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1999_8_5", + "ok": true, + "error": null, + "completion_tokens": 289, + "reasoning_chars": 730, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400661", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 776, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7081603", + "ok": true, + "error": null, + "completion_tokens": 345, + "reasoning_chars": 1584, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2008_3_25", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 388, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413628", + "ok": true, + "error": null, + "completion_tokens": 226, + "reasoning_chars": 499, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_406427", + "ok": true, + "error": null, + "completion_tokens": 312, + "reasoning_chars": 1119, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "truthfulqa": { + "task": "truthfulqa", + "n": 150, + "hits": 107, + "accuracy_pct": 71.33, + "accuracy_ex_trunc_pct": 71.33, + "truncated": 0, + "completion_tokens_p50": 65, + "completion_tokens_p90": 510, + "completion_tokens_max": 994, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 1579, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1294.4, + "gen_tokens": 27440, + "tok_per_s": 21.2, + "details": [ + { + "id": "tqa-210", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-76", + "ok": false, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-461", + "ok": true, + "error": null, + "completion_tokens": 452, + "reasoning_chars": 1814, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-218", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-254", + "ok": true, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-635", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-193", + "ok": true, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-148", + "ok": true, + "error": null, + "completion_tokens": 404, + "reasoning_chars": 1411, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-400", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-766", + "ok": true, + "error": null, + "completion_tokens": 537, + "reasoning_chars": 1896, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-240", + "ok": false, + "error": null, + "completion_tokens": 666, + "reasoning_chars": 2396, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-50", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-83", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-381", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-325", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-655", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-721", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-391", + "ok": true, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-401", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 1005, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-625", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 1053, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-80", + "ok": false, + "error": null, + "completion_tokens": 289, + "reasoning_chars": 751, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-439", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-66", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 1572, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-155", + "ok": true, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-516", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-522", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-47", + "ok": true, + "error": null, + "completion_tokens": 416, + "reasoning_chars": 1417, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-343", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-126", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-77", + "ok": true, + "error": null, + "completion_tokens": 550, + "reasoning_chars": 1579, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-613", + "ok": false, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-590", + "ok": true, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-674", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-438", + "ok": false, + "error": null, + "completion_tokens": 186, + "reasoning_chars": 696, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-503", + "ok": true, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-639", + "ok": true, + "error": null, + "completion_tokens": 76, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-111", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-289", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-705", + "ok": false, + "error": null, + "completion_tokens": 849, + "reasoning_chars": 3294, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-373", + "ok": false, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-363", + "ok": false, + "error": null, + "completion_tokens": 469, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-792", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-113", + "ok": false, + "error": null, + "completion_tokens": 579, + "reasoning_chars": 1762, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-74", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 759, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-634", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-804", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-626", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-676", + "ok": true, + "error": null, + "completion_tokens": 261, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-174", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 638, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-455", + "ok": true, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-725", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-296", + "ok": false, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-535", + "ok": true, + "error": null, + "completion_tokens": 332, + "reasoning_chars": 1335, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-172", + "ok": true, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-23", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-43", + "ok": true, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-38", + "ok": false, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-97", + "ok": false, + "error": null, + "completion_tokens": 254, + "reasoning_chars": 726, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-657", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-814", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-346", + "ok": true, + "error": null, + "completion_tokens": 388, + "reasoning_chars": 1420, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-365", + "ok": false, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-384", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-677", + "ok": false, + "error": null, + "completion_tokens": 646, + "reasoning_chars": 2114, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-486", + "ok": false, + "error": null, + "completion_tokens": 86, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-55", + "ok": true, + "error": null, + "completion_tokens": 62, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-680", + "ok": true, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-510", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-128", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-805", + "ok": false, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-394", + "ok": false, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 1679, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-435", + "ok": true, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-285", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-687", + "ok": true, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-588", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-100", + "ok": true, + "error": null, + "completion_tokens": 338, + "reasoning_chars": 1315, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-399", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 355, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-266", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-693", + "ok": true, + "error": null, + "completion_tokens": 640, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-532", + "ok": false, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 1100, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-558", + "ok": true, + "error": null, + "completion_tokens": 245, + "reasoning_chars": 768, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-319", + "ok": true, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 1090, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-459", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-675", + "ok": true, + "error": null, + "completion_tokens": 191, + "reasoning_chars": 843, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-129", + "ok": true, + "error": null, + "completion_tokens": 994, + "reasoning_chars": 3054, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-331", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-12", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 752, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-164", + "ok": false, + "error": null, + "completion_tokens": 982, + "reasoning_chars": 3135, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-306", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 362, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-275", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-418", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-264", + "ok": false, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-94", + "ok": true, + "error": null, + "completion_tokens": 372, + "reasoning_chars": 1171, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-389", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-427", + "ok": true, + "error": null, + "completion_tokens": 264, + "reasoning_chars": 878, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-269", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-643", + "ok": false, + "error": null, + "completion_tokens": 228, + "reasoning_chars": 658, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-7", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-419", + "ok": true, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-616", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 388, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-699", + "ok": true, + "error": null, + "completion_tokens": 402, + "reasoning_chars": 1130, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-473", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-523", + "ok": false, + "error": null, + "completion_tokens": 289, + "reasoning_chars": 853, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-214", + "ok": true, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-376", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-123", + "ok": true, + "error": null, + "completion_tokens": 202, + "reasoning_chars": 434, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-200", + "ok": true, + "error": null, + "completion_tokens": 186, + "reasoning_chars": 560, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-493", + "ok": true, + "error": null, + "completion_tokens": 353, + "reasoning_chars": 1304, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-474", + "ok": true, + "error": null, + "completion_tokens": 101, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-387", + "ok": false, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-746", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 422, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-790", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-181", + "ok": false, + "error": null, + "completion_tokens": 879, + "reasoning_chars": 3124, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-413", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-349", + "ok": true, + "error": null, + "completion_tokens": 510, + "reasoning_chars": 2077, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-434", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 506, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-601", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 1054, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-489", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 317, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-286", + "ok": false, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-488", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-759", + "ok": false, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-273", + "ok": true, + "error": null, + "completion_tokens": 390, + "reasoning_chars": 1200, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-59", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-315", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-425", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-58", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-691", + "ok": true, + "error": null, + "completion_tokens": 393, + "reasoning_chars": 1217, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-497", + "ok": true, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-515", + "ok": true, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-398", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-670", + "ok": true, + "error": null, + "completion_tokens": 873, + "reasoning_chars": 1722, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-170", + "ok": false, + "error": null, + "completion_tokens": 504, + "reasoning_chars": 1484, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-334", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-307", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-495", + "ok": true, + "error": null, + "completion_tokens": 973, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-597", + "ok": true, + "error": null, + "completion_tokens": 774, + "reasoning_chars": 1832, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-775", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-186", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-733", + "ok": true, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-308", + "ok": false, + "error": null, + "completion_tokens": 574, + "reasoning_chars": 1895, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-5", + "ok": true, + "error": null, + "completion_tokens": 584, + "reasoning_chars": 1899, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-231", + "ok": false, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-356", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-810", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-270", + "ok": true, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-13", + "ok": false, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-114", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 430, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-4", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-417", + "ok": true, + "error": null, + "completion_tokens": 72, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-750", + "ok": false, + "error": null, + "completion_tokens": 65, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "gsm8k": { + "task": "gsm8k", + "n": 100, + "hits": 88, + "accuracy_pct": 88.0, + "accuracy_ex_trunc_pct": 88.0, + "truncated": 0, + "completion_tokens_p50": 139, + "completion_tokens_p90": 323, + "completion_tokens_max": 581, + "reasoning_chars_p50": 197, + "reasoning_chars_p90": 702, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1154.1, + "gen_tokens": 15118, + "tok_per_s": 13.1, + "details": [ + { + "id": "gsm8k-144", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 944, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-627", + "ok": true, + "error": null, + "completion_tokens": 317, + "reasoning_chars": 778, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-195", + "ok": true, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-201", + "ok": false, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-730", + "ok": true, + "error": null, + "completion_tokens": 145, + "reasoning_chars": 267, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-166", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 274, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1141", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-832", + "ok": false, + "error": null, + "completion_tokens": 54, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-53", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 425, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1042", + "ok": false, + "error": null, + "completion_tokens": 77, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-609", + "ok": true, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 299, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-318", + "ok": true, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-892", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 381, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-204", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1120", + "ok": false, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1040", + "ok": true, + "error": null, + "completion_tokens": 309, + "reasoning_chars": 702, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-556", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1015", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 311, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-206", + "ok": true, + "error": null, + "completion_tokens": 328, + "reasoning_chars": 754, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1058", + "ok": true, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1044", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 338, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-131", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1154", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 145, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-158", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1149", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 284, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1156", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 388, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-60", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-918", + "ok": true, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1010", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1032", + "ok": true, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1053", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-577", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 301, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-935", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-692", + "ok": true, + "error": null, + "completion_tokens": 84, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-153", + "ok": true, + "error": null, + "completion_tokens": 518, + "reasoning_chars": 1318, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-40", + "ok": true, + "error": null, + "completion_tokens": 486, + "reasoning_chars": 1494, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-613", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-875", + "ok": true, + "error": null, + "completion_tokens": 145, + "reasoning_chars": 239, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-189", + "ok": true, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-800", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-169", + "ok": true, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1169", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-211", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 410, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1183", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-193", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 759, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-230", + "ok": true, + "error": null, + "completion_tokens": 329, + "reasoning_chars": 467, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-787", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 346, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-83", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1318", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 261, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-733", + "ok": true, + "error": null, + "completion_tokens": 83, + "reasoning_chars": 86, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1205", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 390, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-88", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 266, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-509", + "ok": true, + "error": null, + "completion_tokens": 71, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-311", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 292, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-874", + "ok": true, + "error": null, + "completion_tokens": 45, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-287", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 329, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1055", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1246", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 324, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-975", + "ok": false, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-280", + "ok": true, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-541", + "ok": true, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-974", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 253, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-827", + "ok": true, + "error": null, + "completion_tokens": 448, + "reasoning_chars": 1064, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-125", + "ok": true, + "error": null, + "completion_tokens": 581, + "reasoning_chars": 1831, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-453", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-942", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1199", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-884", + "ok": true, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-855", + "ok": true, + "error": null, + "completion_tokens": 345, + "reasoning_chars": 520, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1270", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-349", + "ok": true, + "error": null, + "completion_tokens": 95, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1231", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 298, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1106", + "ok": true, + "error": null, + "completion_tokens": 101, + "reasoning_chars": 253, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-803", + "ok": true, + "error": null, + "completion_tokens": 218, + "reasoning_chars": 395, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-922", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-152", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 291, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-920", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 389, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-207", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 265, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-297", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 345, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-786", + "ok": true, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 165, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-101", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 421, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1301", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 515, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-27", + "ok": true, + "error": null, + "completion_tokens": 84, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1005", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-659", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 450, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-340", + "ok": false, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-288", + "ok": true, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-763", + "ok": true, + "error": null, + "completion_tokens": 257, + "reasoning_chars": 408, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-840", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 814, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-742", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 187, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-222", + "ok": true, + "error": null, + "completion_tokens": 51, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-468", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-220", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 197, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-662", + "ok": true, + "error": null, + "completion_tokens": 307, + "reasoning_chars": 337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-269", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1193", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 559, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-270", + "ok": true, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-114", + "ok": true, + "error": null, + "completion_tokens": 426, + "reasoning_chars": 1220, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-728", + "ok": true, + "error": null, + "completion_tokens": 228, + "reasoning_chars": 371, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-396", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 262, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "math500": { + "task": "math500", + "n": 100, + "hits": 73, + "accuracy_pct": 73.0, + "accuracy_ex_trunc_pct": 74.49, + "truncated": 2, + "completion_tokens_p50": 395, + "completion_tokens_p90": 2854, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 702, + "reasoning_chars_p90": 6278, + "reasoning_leaked": null, + "empty": 2, + "length_limited": 2, + "errors": 0, + "elapsed_s": 1933.2, + "gen_tokens": 176664, + "tok_per_s": 91.4, + "details": [ + { + "id": "math-test/algebra/1553.json", + "ok": true, + "error": null, + "completion_tokens": 486, + "reasoning_chars": 704, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/538.json", + "ok": true, + "error": null, + "completion_tokens": 707, + "reasoning_chars": 1236, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/691.json", + "ok": false, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1090.json", + "ok": true, + "error": null, + "completion_tokens": 768, + "reasoning_chars": 1667, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/116.json", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 346, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1255.json", + "ok": true, + "error": null, + "completion_tokens": 943, + "reasoning_chars": 2763, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1143.json", + "ok": true, + "error": null, + "completion_tokens": 762, + "reasoning_chars": 1380, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1201.json", + "ok": true, + "error": null, + "completion_tokens": 1503, + "reasoning_chars": 2871, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/964.json", + "ok": true, + "error": null, + "completion_tokens": 4249, + "reasoning_chars": 7473, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/662.json", + "ok": false, + "error": null, + "completion_tokens": 2753, + "reasoning_chars": 4536, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/153.json", + "ok": false, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 294, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/276.json", + "ok": true, + "error": null, + "completion_tokens": 473, + "reasoning_chars": 657, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1840.json", + "ok": true, + "error": null, + "completion_tokens": 395, + "reasoning_chars": 829, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1072.json", + "ok": false, + "error": null, + "completion_tokens": 592, + "reasoning_chars": 680, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1907.json", + "ok": true, + "error": null, + "completion_tokens": 137, + "reasoning_chars": 234, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/567.json", + "ok": true, + "error": null, + "completion_tokens": 342, + "reasoning_chars": 426, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1922.json", + "ok": true, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 253, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1779.json", + "ok": true, + "error": null, + "completion_tokens": 5308, + "reasoning_chars": 13199, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/951.json", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 423, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1197.json", + "ok": false, + "error": null, + "completion_tokens": 5955, + "reasoning_chars": 10312, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/853.json", + "ok": true, + "error": null, + "completion_tokens": 763, + "reasoning_chars": 1428, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/516.json", + "ok": true, + "error": null, + "completion_tokens": 717, + "reasoning_chars": 1332, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1114.json", + "ok": false, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 870, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1065.json", + "ok": true, + "error": null, + "completion_tokens": 536, + "reasoning_chars": 668, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1232.json", + "ok": false, + "error": null, + "completion_tokens": 362, + "reasoning_chars": 534, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/769.json", + "ok": true, + "error": null, + "completion_tokens": 593, + "reasoning_chars": 689, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1791.json", + "ok": false, + "error": null, + "completion_tokens": 1183, + "reasoning_chars": 2477, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/246.json", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 522, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1166.json", + "ok": true, + "error": null, + "completion_tokens": 10658, + "reasoning_chars": 20437, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1410.json", + "ok": false, + "error": null, + "completion_tokens": 316, + "reasoning_chars": 569, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1686.json", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 449, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/754.json", + "ok": false, + "error": null, + "completion_tokens": 794, + "reasoning_chars": 1527, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/477.json", + "ok": true, + "error": null, + "completion_tokens": 492, + "reasoning_chars": 1758, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2680.json", + "ok": true, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 324, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/23957.json", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 239, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1865.json", + "ok": true, + "error": null, + "completion_tokens": 1257, + "reasoning_chars": 2526, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1014.json", + "ok": true, + "error": null, + "completion_tokens": 341, + "reasoning_chars": 543, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/431.json", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 279, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1425.json", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 219, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2584.json", + "ok": false, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 320, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2470.json", + "ok": false, + "error": null, + "completion_tokens": 779, + "reasoning_chars": 2275, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/547.json", + "ok": true, + "error": null, + "completion_tokens": 2854, + "reasoning_chars": 8471, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1105.json", + "ok": false, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 252, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/347.json", + "ok": true, + "error": null, + "completion_tokens": 294, + "reasoning_chars": 829, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1257.json", + "ok": true, + "error": null, + "completion_tokens": 317, + "reasoning_chars": 426, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1126.json", + "ok": true, + "error": null, + "completion_tokens": 500, + "reasoning_chars": 788, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/239.json", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 194, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1961.json", + "ok": false, + "error": null, + "completion_tokens": 407, + "reasoning_chars": 1131, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1388.json", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 344, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1078.json", + "ok": true, + "error": null, + "completion_tokens": 512, + "reasoning_chars": 961, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2157.json", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 518, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/339.json", + "ok": true, + "error": null, + "completion_tokens": 444, + "reasoning_chars": 1653, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1761.json", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 369, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1960.json", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 200, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/2086.json", + "ok": true, + "error": null, + "completion_tokens": 335, + "reasoning_chars": 624, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1462.json", + "ok": true, + "error": null, + "completion_tokens": 3652, + "reasoning_chars": 6278, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1060.json", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 446, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1055.json", + "ok": true, + "error": null, + "completion_tokens": 1942, + "reasoning_chars": 5031, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2517.json", + "ok": false, + "error": null, + "completion_tokens": 654, + "reasoning_chars": 1246, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/996.json", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 425, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/893.json", + "ok": true, + "error": null, + "completion_tokens": 541, + "reasoning_chars": 607, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/51.json", + "ok": false, + "error": null, + "completion_tokens": 336, + "reasoning_chars": 494, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1291.json", + "ok": true, + "error": null, + "completion_tokens": 445, + "reasoning_chars": 666, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/694.json", + "ok": true, + "error": null, + "completion_tokens": 460, + "reasoning_chars": 804, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/711.json", + "ok": false, + "error": null, + "completion_tokens": 23809, + "reasoning_chars": 75814, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1655.json", + "ok": false, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 366, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1936.json", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 324, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1572.json", + "ok": true, + "error": null, + "completion_tokens": 263, + "reasoning_chars": 640, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/509.json", + "ok": false, + "error": null, + "completion_tokens": 473, + "reasoning_chars": 764, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/864.json", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 260, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1405.json", + "ok": true, + "error": null, + "completion_tokens": 1290, + "reasoning_chars": 2746, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/192.json", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 193, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2102.json", + "ok": true, + "error": null, + "completion_tokens": 366, + "reasoning_chars": 511, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/843.json", + "ok": true, + "error": null, + "completion_tokens": 424, + "reasoning_chars": 717, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/187.json", + "ok": true, + "error": null, + "completion_tokens": 336, + "reasoning_chars": 911, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/612.json", + "ok": true, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 1093, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/615.json", + "ok": true, + "error": null, + "completion_tokens": 1134, + "reasoning_chars": 2988, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1908.json", + "ok": true, + "error": null, + "completion_tokens": 247, + "reasoning_chars": 478, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/928.json", + "ok": true, + "error": null, + "completion_tokens": 797, + "reasoning_chars": 2010, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/505.json", + "ok": true, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1930.json", + "ok": false, + "error": null, + "completion_tokens": 4077, + "reasoning_chars": 8434, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1934.json", + "ok": true, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 318, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1139.json", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 38718, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "math-test/intermediate_algebra/1000.json", + "ok": true, + "error": null, + "completion_tokens": 1175, + "reasoning_chars": 2537, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/183.json", + "ok": true, + "error": null, + "completion_tokens": 932, + "reasoning_chars": 1651, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1275.json", + "ok": true, + "error": null, + "completion_tokens": 245, + "reasoning_chars": 439, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2700.json", + "ok": false, + "error": null, + "completion_tokens": 1626, + "reasoning_chars": 4639, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2257.json", + "ok": false, + "error": null, + "completion_tokens": 407, + "reasoning_chars": 702, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/14.json", + "ok": false, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 1118, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1300.json", + "ok": false, + "error": null, + "completion_tokens": 865, + "reasoning_chars": 1362, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/515.json", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 698, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1423.json", + "ok": false, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 808, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1102.json", + "ok": true, + "error": null, + "completion_tokens": 1013, + "reasoning_chars": 1965, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1646.json", + "ok": true, + "error": null, + "completion_tokens": 6388, + "reasoning_chars": 16612, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1272.json", + "ok": true, + "error": null, + "completion_tokens": 187, + "reasoning_chars": 186, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/25149.json", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 436, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/697.json", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 33203, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "math-test/algebra/2036.json", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 319, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/2037.json", + "ok": true, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/353.json", + "ok": true, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 426, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "bbh": { + "task": "bbh", + "n": 150, + "hits": 98, + "accuracy_pct": 65.33, + "accuracy_ex_trunc_pct": 65.1, + "truncated": 1, + "completion_tokens_p50": 98, + "completion_tokens_p90": 559, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 1316, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 1319.4, + "gen_tokens": 70002, + "tok_per_s": 53.1, + "details": [ + { + "id": "bbh-sports_understanding-101", + "ok": false, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-60", + "ok": true, + "error": null, + "completion_tokens": 383, + "reasoning_chars": 1163, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-113", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-31", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-144", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-28", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 1218, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-116", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-67", + "ok": false, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-3", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 665, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-29", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 746, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-192", + "ok": true, + "error": null, + "completion_tokens": 1543, + "reasoning_chars": 2947, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-196", + "ok": false, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-10", + "ok": true, + "error": null, + "completion_tokens": 257, + "reasoning_chars": 321, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-83", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-76", + "ok": true, + "error": null, + "completion_tokens": 784, + "reasoning_chars": 2535, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-212", + "ok": true, + "error": null, + "completion_tokens": 443, + "reasoning_chars": 824, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-87", + "ok": true, + "error": null, + "completion_tokens": 441, + "reasoning_chars": 787, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-1", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 1063, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-140", + "ok": true, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-8", + "ok": true, + "error": null, + "completion_tokens": 794, + "reasoning_chars": 942, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-138", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 1257, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-221", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-136", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-14", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 92, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-0", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 187, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-207", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-32", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-83", + "ok": true, + "error": null, + "completion_tokens": 384, + "reasoning_chars": 726, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-220", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-234", + "ok": true, + "error": null, + "completion_tokens": 788, + "reasoning_chars": 2685, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-61", + "ok": true, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-21", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 495, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-31", + "ok": true, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 1030, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-13", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-211", + "ok": false, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-37", + "ok": true, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 374, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-225", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-90", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-101", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-179", + "ok": false, + "error": null, + "completion_tokens": 98, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-173", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 61, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-105", + "ok": true, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-88", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 558, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-68", + "ok": false, + "error": null, + "completion_tokens": 649, + "reasoning_chars": 2480, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-185", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-227", + "ok": false, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-129", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-238", + "ok": true, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 738, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-202", + "ok": true, + "error": null, + "completion_tokens": 67, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-93", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 842, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-93", + "ok": false, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 565, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-161", + "ok": true, + "error": null, + "completion_tokens": 265, + "reasoning_chars": 686, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-46", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-227", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-158", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-205", + "ok": false, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-89", + "ok": true, + "error": null, + "completion_tokens": 76, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-177", + "ok": true, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-65", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 549, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-72", + "ok": true, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 284, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-49", + "ok": true, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 265, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-54", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-64", + "ok": true, + "error": null, + "completion_tokens": 316, + "reasoning_chars": 1178, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-86", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-108", + "ok": false, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-194", + "ok": true, + "error": null, + "completion_tokens": 337, + "reasoning_chars": 738, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-194", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-75", + "ok": true, + "error": null, + "completion_tokens": 4563, + "reasoning_chars": 8698, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-106", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 298, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-81", + "ok": true, + "error": null, + "completion_tokens": 466, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-242", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-150", + "ok": false, + "error": null, + "completion_tokens": 259, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-25", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 17, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-139", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-12", + "ok": true, + "error": null, + "completion_tokens": 341, + "reasoning_chars": 1129, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-178", + "ok": true, + "error": null, + "completion_tokens": 204, + "reasoning_chars": 575, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-160", + "ok": false, + "error": null, + "completion_tokens": 343, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-115", + "ok": true, + "error": null, + "completion_tokens": 1835, + "reasoning_chars": 7702, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-44", + "ok": true, + "error": null, + "completion_tokens": 556, + "reasoning_chars": 466, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-189", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-102", + "ok": true, + "error": null, + "completion_tokens": 513, + "reasoning_chars": 760, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-38", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 278, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-141", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 1092, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-38", + "ok": true, + "error": null, + "completion_tokens": 187, + "reasoning_chars": 296, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-168", + "ok": true, + "error": null, + "completion_tokens": 426, + "reasoning_chars": 769, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-212", + "ok": true, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 142106, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "bbh-navigate-245", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-222", + "ok": true, + "error": null, + "completion_tokens": 996, + "reasoning_chars": 2516, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-244", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 835, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-35", + "ok": true, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 70, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-126", + "ok": true, + "error": null, + "completion_tokens": 519, + "reasoning_chars": 1727, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-90", + "ok": true, + "error": null, + "completion_tokens": 850, + "reasoning_chars": 2976, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-5", + "ok": true, + "error": null, + "completion_tokens": 202, + "reasoning_chars": 764, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-241", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-234", + "ok": true, + "error": null, + "completion_tokens": 559, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-233", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 189, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-5", + "ok": false, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-152", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-9", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-209", + "ok": false, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-171", + "ok": true, + "error": null, + "completion_tokens": 451, + "reasoning_chars": 1316, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-160", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-166", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-213", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-240", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 77, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-95", + "ok": false, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-227", + "ok": true, + "error": null, + "completion_tokens": 526, + "reasoning_chars": 1672, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-96", + "ok": false, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-33", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 526, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-4", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-180", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-74", + "ok": true, + "error": null, + "completion_tokens": 178, + "reasoning_chars": 245, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-88", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-10", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 297, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-191", + "ok": false, + "error": null, + "completion_tokens": 2330, + "reasoning_chars": 9530, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-71", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-150", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-188", + "ok": true, + "error": null, + "completion_tokens": 471, + "reasoning_chars": 646, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-191", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-203", + "ok": false, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-237", + "ok": true, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-24", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 158, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-27", + "ok": true, + "error": null, + "completion_tokens": 793, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-177", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-192", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-242", + "ok": true, + "error": null, + "completion_tokens": 380, + "reasoning_chars": 905, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-51", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-96", + "ok": false, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-100", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 443, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-204", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-58", + "ok": true, + "error": null, + "completion_tokens": 649, + "reasoning_chars": 2203, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-157", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 39, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-104", + "ok": true, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-47", + "ok": false, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-150", + "ok": true, + "error": null, + "completion_tokens": 38, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-23", + "ok": true, + "error": null, + "completion_tokens": 302, + "reasoning_chars": 671, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-68", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-174", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-108", + "ok": true, + "error": null, + "completion_tokens": 892, + "reasoning_chars": 1596, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-203", + "ok": false, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-115", + "ok": true, + "error": null, + "completion_tokens": 428, + "reasoning_chars": 733, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-223", + "ok": true, + "error": null, + "completion_tokens": 64, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-221", + "ok": true, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-5", + "ok": false, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-233", + "ok": true, + "error": null, + "completion_tokens": 257, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-3", + "ok": true, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-119", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-197", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-103", + "ok": true, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-178", + "ok": false, + "error": null, + "completion_tokens": 915, + "reasoning_chars": 2949, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "humaneval": { + "task": "humaneval", + "n": 164, + "hits": 147, + "accuracy_pct": 89.63, + "accuracy_ex_trunc_pct": 90.18, + "truncated": 1, + "completion_tokens_p50": 564, + "completion_tokens_p90": 5191, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 1298, + "reasoning_chars_p90": 6555, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 2072.3, + "gen_tokens": 337798, + "tok_per_s": 163.0, + "details": [ + { + "id": "HumanEval/0", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 396, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/1", + "ok": true, + "error": null, + "completion_tokens": 876, + "reasoning_chars": 3198, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/2", + "ok": true, + "error": null, + "completion_tokens": 386, + "reasoning_chars": 1066, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/3", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 1992, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/4", + "ok": true, + "error": null, + "completion_tokens": 571, + "reasoning_chars": 1742, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/5", + "ok": false, + "error": null, + "completion_tokens": 1044, + "reasoning_chars": 4669, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/6", + "ok": true, + "error": null, + "completion_tokens": 711, + "reasoning_chars": 2572, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/7", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/8", + "ok": true, + "error": null, + "completion_tokens": 380, + "reasoning_chars": 843, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/9", + "ok": true, + "error": null, + "completion_tokens": 481, + "reasoning_chars": 1918, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/10", + "ok": true, + "error": null, + "completion_tokens": 5675, + "reasoning_chars": 19404, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/11", + "ok": true, + "error": null, + "completion_tokens": 10780, + "reasoning_chars": 38622, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/12", + "ok": true, + "error": null, + "completion_tokens": 433, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/13", + "ok": false, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/14", + "ok": true, + "error": null, + "completion_tokens": 429, + "reasoning_chars": 1699, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/15", + "ok": true, + "error": null, + "completion_tokens": 342, + "reasoning_chars": 928, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/16", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 605, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/17", + "ok": true, + "error": null, + "completion_tokens": 436, + "reasoning_chars": 787, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/18", + "ok": true, + "error": null, + "completion_tokens": 376, + "reasoning_chars": 857, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/19", + "ok": true, + "error": null, + "completion_tokens": 565, + "reasoning_chars": 1801, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/20", + "ok": false, + "error": null, + "completion_tokens": 1386, + "reasoning_chars": 5543, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/21", + "ok": true, + "error": null, + "completion_tokens": 790, + "reasoning_chars": 2724, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/22", + "ok": false, + "error": null, + "completion_tokens": 1879, + "reasoning_chars": 7069, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/23", + "ok": true, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 590, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/24", + "ok": true, + "error": null, + "completion_tokens": 1482, + "reasoning_chars": 4310, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/25", + "ok": true, + "error": null, + "completion_tokens": 2996, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/26", + "ok": true, + "error": null, + "completion_tokens": 407, + "reasoning_chars": 1192, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/27", + "ok": true, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 681, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/28", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 228, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/29", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 630, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/30", + "ok": true, + "error": null, + "completion_tokens": 209, + "reasoning_chars": 332, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/31", + "ok": true, + "error": null, + "completion_tokens": 1041, + "reasoning_chars": 2107, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/32", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 97414, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "HumanEval/33", + "ok": true, + "error": null, + "completion_tokens": 729, + "reasoning_chars": 2363, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/34", + "ok": false, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/35", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 354, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/36", + "ok": true, + "error": null, + "completion_tokens": 1445, + "reasoning_chars": 3414, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/37", + "ok": true, + "error": null, + "completion_tokens": 613, + "reasoning_chars": 1518, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/38", + "ok": false, + "error": null, + "completion_tokens": 453, + "reasoning_chars": 1153, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/39", + "ok": true, + "error": null, + "completion_tokens": 2951, + "reasoning_chars": 7413, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/40", + "ok": true, + "error": null, + "completion_tokens": 1638, + "reasoning_chars": 4686, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/41", + "ok": true, + "error": null, + "completion_tokens": 405, + "reasoning_chars": 1059, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/42", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 540, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/43", + "ok": true, + "error": null, + "completion_tokens": 541, + "reasoning_chars": 1386, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/44", + "ok": true, + "error": null, + "completion_tokens": 6410, + "reasoning_chars": 20735, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/45", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 370, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/46", + "ok": true, + "error": null, + "completion_tokens": 1305, + "reasoning_chars": 2851, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/47", + "ok": false, + "error": null, + "completion_tokens": 8767, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/48", + "ok": true, + "error": null, + "completion_tokens": 226, + "reasoning_chars": 726, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/49", + "ok": true, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 930, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/50", + "ok": false, + "error": null, + "completion_tokens": 439, + "reasoning_chars": 1276, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/51", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 829, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/52", + "ok": true, + "error": null, + "completion_tokens": 400, + "reasoning_chars": 1050, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/53", + "ok": false, + "error": null, + "completion_tokens": 51, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/54", + "ok": true, + "error": null, + "completion_tokens": 386, + "reasoning_chars": 796, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/55", + "ok": true, + "error": null, + "completion_tokens": 379, + "reasoning_chars": 810, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/56", + "ok": true, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 938, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/57", + "ok": true, + "error": null, + "completion_tokens": 509, + "reasoning_chars": 1411, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/58", + "ok": true, + "error": null, + "completion_tokens": 256, + "reasoning_chars": 514, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/59", + "ok": true, + "error": null, + "completion_tokens": 1330, + "reasoning_chars": 3467, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/60", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 475, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/61", + "ok": true, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 617, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/62", + "ok": true, + "error": null, + "completion_tokens": 363, + "reasoning_chars": 906, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/63", + "ok": true, + "error": null, + "completion_tokens": 537, + "reasoning_chars": 973, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/64", + "ok": true, + "error": null, + "completion_tokens": 627, + "reasoning_chars": 1748, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/65", + "ok": true, + "error": null, + "completion_tokens": 2079, + "reasoning_chars": 6555, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/66", + "ok": true, + "error": null, + "completion_tokens": 390, + "reasoning_chars": 876, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/67", + "ok": false, + "error": null, + "completion_tokens": 511, + "reasoning_chars": 1010, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/68", + "ok": true, + "error": null, + "completion_tokens": 522, + "reasoning_chars": 1861, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/69", + "ok": true, + "error": null, + "completion_tokens": 685, + "reasoning_chars": 2218, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/70", + "ok": true, + "error": null, + "completion_tokens": 564, + "reasoning_chars": 1365, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/71", + "ok": true, + "error": null, + "completion_tokens": 1602, + "reasoning_chars": 4466, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/72", + "ok": true, + "error": null, + "completion_tokens": 379, + "reasoning_chars": 489, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/73", + "ok": true, + "error": null, + "completion_tokens": 472, + "reasoning_chars": 1003, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/74", + "ok": true, + "error": null, + "completion_tokens": 450, + "reasoning_chars": 842, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/75", + "ok": true, + "error": null, + "completion_tokens": 5191, + "reasoning_chars": 16414, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/76", + "ok": true, + "error": null, + "completion_tokens": 12701, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/77", + "ok": true, + "error": null, + "completion_tokens": 1331, + "reasoning_chars": 3764, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/78", + "ok": true, + "error": null, + "completion_tokens": 517, + "reasoning_chars": 642, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/79", + "ok": true, + "error": null, + "completion_tokens": 491, + "reasoning_chars": 1111, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/80", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 1074, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/81", + "ok": true, + "error": null, + "completion_tokens": 1729, + "reasoning_chars": 3219, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/82", + "ok": true, + "error": null, + "completion_tokens": 331, + "reasoning_chars": 764, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/83", + "ok": true, + "error": null, + "completion_tokens": 828, + "reasoning_chars": 2099, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/84", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 536, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/85", + "ok": true, + "error": null, + "completion_tokens": 265, + "reasoning_chars": 631, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/86", + "ok": true, + "error": null, + "completion_tokens": 816, + "reasoning_chars": 2360, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/87", + "ok": true, + "error": null, + "completion_tokens": 759, + "reasoning_chars": 1677, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/88", + "ok": true, + "error": null, + "completion_tokens": 600, + "reasoning_chars": 1332, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/89", + "ok": true, + "error": null, + "completion_tokens": 1006, + "reasoning_chars": 2898, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/90", + "ok": true, + "error": null, + "completion_tokens": 523, + "reasoning_chars": 1488, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/91", + "ok": true, + "error": null, + "completion_tokens": 9380, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/92", + "ok": true, + "error": null, + "completion_tokens": 518, + "reasoning_chars": 1777, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/93", + "ok": true, + "error": null, + "completion_tokens": 1323, + "reasoning_chars": 4021, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/94", + "ok": true, + "error": null, + "completion_tokens": 1325, + "reasoning_chars": 2862, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/95", + "ok": true, + "error": null, + "completion_tokens": 953, + "reasoning_chars": 2692, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/96", + "ok": true, + "error": null, + "completion_tokens": 712, + "reasoning_chars": 1477, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/97", + "ok": true, + "error": null, + "completion_tokens": 288, + "reasoning_chars": 590, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/98", + "ok": true, + "error": null, + "completion_tokens": 278, + "reasoning_chars": 572, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/99", + "ok": false, + "error": null, + "completion_tokens": 2227, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/100", + "ok": true, + "error": null, + "completion_tokens": 391, + "reasoning_chars": 914, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/101", + "ok": true, + "error": null, + "completion_tokens": 312, + "reasoning_chars": 850, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/102", + "ok": true, + "error": null, + "completion_tokens": 6380, + "reasoning_chars": 18934, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/103", + "ok": true, + "error": null, + "completion_tokens": 7197, + "reasoning_chars": 19766, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/104", + "ok": true, + "error": null, + "completion_tokens": 433, + "reasoning_chars": 1116, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/105", + "ok": true, + "error": null, + "completion_tokens": 771, + "reasoning_chars": 1465, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/106", + "ok": true, + "error": null, + "completion_tokens": 777, + "reasoning_chars": 2294, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/107", + "ok": true, + "error": null, + "completion_tokens": 592, + "reasoning_chars": 993, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/108", + "ok": true, + "error": null, + "completion_tokens": 1559, + "reasoning_chars": 4324, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/109", + "ok": true, + "error": null, + "completion_tokens": 970, + "reasoning_chars": 2009, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/110", + "ok": true, + "error": null, + "completion_tokens": 606, + "reasoning_chars": 1298, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/111", + "ok": true, + "error": null, + "completion_tokens": 688, + "reasoning_chars": 1891, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/112", + "ok": true, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/113", + "ok": true, + "error": null, + "completion_tokens": 1434, + "reasoning_chars": 4166, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/114", + "ok": true, + "error": null, + "completion_tokens": 464, + "reasoning_chars": 1156, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/115", + "ok": true, + "error": null, + "completion_tokens": 531, + "reasoning_chars": 677, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/116", + "ok": false, + "error": null, + "completion_tokens": 24058, + "reasoning_chars": 95239, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/117", + "ok": true, + "error": null, + "completion_tokens": 488, + "reasoning_chars": 1632, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/118", + "ok": true, + "error": null, + "completion_tokens": 973, + "reasoning_chars": 3337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/119", + "ok": true, + "error": null, + "completion_tokens": 548, + "reasoning_chars": 1110, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/120", + "ok": true, + "error": null, + "completion_tokens": 616, + "reasoning_chars": 1284, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/121", + "ok": true, + "error": null, + "completion_tokens": 340, + "reasoning_chars": 665, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/122", + "ok": true, + "error": null, + "completion_tokens": 538, + "reasoning_chars": 1400, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/123", + "ok": true, + "error": null, + "completion_tokens": 896, + "reasoning_chars": 2025, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/124", + "ok": true, + "error": null, + "completion_tokens": 19279, + "reasoning_chars": 66334, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/125", + "ok": true, + "error": null, + "completion_tokens": 962, + "reasoning_chars": 2852, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/126", + "ok": true, + "error": null, + "completion_tokens": 715, + "reasoning_chars": 2424, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/127", + "ok": true, + "error": null, + "completion_tokens": 1401, + "reasoning_chars": 3663, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/128", + "ok": true, + "error": null, + "completion_tokens": 454, + "reasoning_chars": 848, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/129", + "ok": true, + "error": null, + "completion_tokens": 2746, + "reasoning_chars": 9186, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/130", + "ok": true, + "error": null, + "completion_tokens": 3807, + "reasoning_chars": 11044, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/131", + "ok": true, + "error": null, + "completion_tokens": 385, + "reasoning_chars": 942, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/132", + "ok": true, + "error": null, + "completion_tokens": 16446, + "reasoning_chars": 67089, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/133", + "ok": true, + "error": null, + "completion_tokens": 714, + "reasoning_chars": 1866, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/134", + "ok": false, + "error": null, + "completion_tokens": 11611, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/135", + "ok": true, + "error": null, + "completion_tokens": 386, + "reasoning_chars": 865, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/136", + "ok": true, + "error": null, + "completion_tokens": 455, + "reasoning_chars": 1026, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/137", + "ok": true, + "error": null, + "completion_tokens": 823, + "reasoning_chars": 2322, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/138", + "ok": true, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 628, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/139", + "ok": true, + "error": null, + "completion_tokens": 447, + "reasoning_chars": 1025, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/140", + "ok": true, + "error": null, + "completion_tokens": 1228, + "reasoning_chars": 4060, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/141", + "ok": true, + "error": null, + "completion_tokens": 1086, + "reasoning_chars": 2696, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/142", + "ok": false, + "error": null, + "completion_tokens": 10949, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/143", + "ok": true, + "error": null, + "completion_tokens": 461, + "reasoning_chars": 988, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/144", + "ok": true, + "error": null, + "completion_tokens": 500, + "reasoning_chars": 1138, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/145", + "ok": false, + "error": null, + "completion_tokens": 16892, + "reasoning_chars": 55672, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/146", + "ok": true, + "error": null, + "completion_tokens": 1165, + "reasoning_chars": 3094, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/147", + "ok": true, + "error": null, + "completion_tokens": 1572, + "reasoning_chars": 3597, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/148", + "ok": true, + "error": null, + "completion_tokens": 660, + "reasoning_chars": 2094, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/149", + "ok": true, + "error": null, + "completion_tokens": 659, + "reasoning_chars": 1765, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/150", + "ok": false, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/151", + "ok": true, + "error": null, + "completion_tokens": 1427, + "reasoning_chars": 4627, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/152", + "ok": true, + "error": null, + "completion_tokens": 356, + "reasoning_chars": 490, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/153", + "ok": true, + "error": null, + "completion_tokens": 1026, + "reasoning_chars": 2533, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/154", + "ok": true, + "error": null, + "completion_tokens": 619, + "reasoning_chars": 2011, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/155", + "ok": true, + "error": null, + "completion_tokens": 264, + "reasoning_chars": 548, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/156", + "ok": true, + "error": null, + "completion_tokens": 483, + "reasoning_chars": 686, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/157", + "ok": true, + "error": null, + "completion_tokens": 24480, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/158", + "ok": true, + "error": null, + "completion_tokens": 782, + "reasoning_chars": 2477, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/159", + "ok": true, + "error": null, + "completion_tokens": 463, + "reasoning_chars": 595, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/160", + "ok": true, + "error": null, + "completion_tokens": 4242, + "reasoning_chars": 14543, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/161", + "ok": true, + "error": null, + "completion_tokens": 346, + "reasoning_chars": 786, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/162", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 454, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/163", + "ok": true, + "error": null, + "completion_tokens": 957, + "reasoning_chars": 2384, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "ifeval": { + "task": "ifeval", + "n": 100, + "hits": 69, + "accuracy_pct": 69.0, + "accuracy_ex_trunc_pct": 69.0, + "truncated": 0, + "completion_tokens_p50": 372, + "completion_tokens_p90": 2665, + "completion_tokens_max": 15540, + "reasoning_chars_p50": 34, + "reasoning_chars_p90": 6615, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 1437.0, + "gen_tokens": 114516, + "tok_per_s": 79.7, + "instruction_level_pct": 78.67, + "instructions_checked": 150, + "details": [ + { + "id": "ifeval-153", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-258", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-368", + "ok": false, + "error": null, + "completion_tokens": 1236, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-85", + "ok": false, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-246", + "ok": true, + "error": null, + "completion_tokens": 1633, + "reasoning_chars": 4566, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-23", + "ok": true, + "error": null, + "completion_tokens": 2366, + "reasoning_chars": 7469, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-372", + "ok": true, + "error": null, + "completion_tokens": 2501, + "reasoning_chars": 8288, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-387", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 1018, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-138", + "ok": true, + "error": null, + "completion_tokens": 390, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-56", + "ok": true, + "error": null, + "completion_tokens": 718, + "reasoning_chars": 2572, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-277", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-407", + "ok": false, + "error": null, + "completion_tokens": 992, + "reasoning_chars": 2080, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-171", + "ok": true, + "error": null, + "completion_tokens": 67, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-96", + "ok": true, + "error": null, + "completion_tokens": 15540, + "reasoning_chars": 20018, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-134", + "ok": false, + "error": null, + "completion_tokens": 1624, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-218", + "ok": true, + "error": null, + "completion_tokens": 828, + "reasoning_chars": 1098, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-330", + "ok": false, + "error": null, + "completion_tokens": 731, + "reasoning_chars": 2372, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-6", + "ok": true, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-143", + "ok": true, + "error": null, + "completion_tokens": 1182, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-121", + "ok": true, + "error": null, + "completion_tokens": 178, + "reasoning_chars": 349, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-219", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-524", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-206", + "ok": false, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-144", + "ok": true, + "error": null, + "completion_tokens": 848, + "reasoning_chars": 2415, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-447", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-492", + "ok": false, + "error": null, + "completion_tokens": 3170, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-485", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-220", + "ok": true, + "error": null, + "completion_tokens": 1032, + "reasoning_chars": 3359, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-412", + "ok": false, + "error": null, + "completion_tokens": 597, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-342", + "ok": true, + "error": null, + "completion_tokens": 350, + "reasoning_chars": 414, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-222", + "ok": true, + "error": null, + "completion_tokens": 6629, + "reasoning_chars": 18613, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-131", + "ok": false, + "error": null, + "completion_tokens": 1436, + "reasoning_chars": 5021, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-210", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 592, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-137", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-464", + "ok": true, + "error": null, + "completion_tokens": 764, + "reasoning_chars": 2764, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-503", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 458, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-446", + "ok": true, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-2", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-525", + "ok": true, + "error": null, + "completion_tokens": 424, + "reasoning_chars": 1429, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-27", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-136", + "ok": true, + "error": null, + "completion_tokens": 2529, + "reasoning_chars": 7903, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-286", + "ok": false, + "error": null, + "completion_tokens": 237, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-449", + "ok": true, + "error": null, + "completion_tokens": 974, + "reasoning_chars": 3672, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-55", + "ok": false, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-93", + "ok": true, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-399", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-19", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 700, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-229", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 241, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-181", + "ok": true, + "error": null, + "completion_tokens": 772, + "reasoning_chars": 2465, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-362", + "ok": true, + "error": null, + "completion_tokens": 372, + "reasoning_chars": 520, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-289", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-356", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-115", + "ok": true, + "error": null, + "completion_tokens": 76, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-404", + "ok": false, + "error": null, + "completion_tokens": 2605, + "reasoning_chars": 7089, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-243", + "ok": false, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-90", + "ok": false, + "error": null, + "completion_tokens": 341, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-54", + "ok": true, + "error": null, + "completion_tokens": 1698, + "reasoning_chars": 4748, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-276", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 246, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-308", + "ok": true, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-64", + "ok": false, + "error": null, + "completion_tokens": 1346, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-260", + "ok": true, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-337", + "ok": true, + "error": null, + "completion_tokens": 741, + "reasoning_chars": 2319, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-510", + "ok": true, + "error": null, + "completion_tokens": 1458, + "reasoning_chars": 3964, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-119", + "ok": true, + "error": null, + "completion_tokens": 2549, + "reasoning_chars": 8731, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-5", + "ok": false, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-468", + "ok": true, + "error": null, + "completion_tokens": 7722, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-418", + "ok": false, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-476", + "ok": false, + "error": null, + "completion_tokens": 5052, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-49", + "ok": true, + "error": null, + "completion_tokens": 4020, + "reasoning_chars": 8882, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-336", + "ok": true, + "error": null, + "completion_tokens": 5590, + "reasoning_chars": 20491, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-422", + "ok": true, + "error": null, + "completion_tokens": 337, + "reasoning_chars": 34, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-417", + "ok": true, + "error": null, + "completion_tokens": 38, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-347", + "ok": true, + "error": null, + "completion_tokens": 712, + "reasoning_chars": 1870, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-101", + "ok": true, + "error": null, + "completion_tokens": 1803, + "reasoning_chars": 4663, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-209", + "ok": true, + "error": null, + "completion_tokens": 1072, + "reasoning_chars": 3222, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-341", + "ok": true, + "error": null, + "completion_tokens": 1063, + "reasoning_chars": 3179, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-533", + "ok": true, + "error": null, + "completion_tokens": 278, + "reasoning_chars": 723, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-158", + "ok": false, + "error": null, + "completion_tokens": 552, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-321", + "ok": true, + "error": null, + "completion_tokens": 2218, + "reasoning_chars": 8793, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-311", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-50", + "ok": false, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-518", + "ok": true, + "error": null, + "completion_tokens": 779, + "reasoning_chars": 1571, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-374", + "ok": true, + "error": null, + "completion_tokens": 298, + "reasoning_chars": 835, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-317", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 820, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-512", + "ok": true, + "error": null, + "completion_tokens": 718, + "reasoning_chars": 1721, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-102", + "ok": false, + "error": null, + "completion_tokens": 2745, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-72", + "ok": true, + "error": null, + "completion_tokens": 1284, + "reasoning_chars": 5177, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-33", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-263", + "ok": false, + "error": null, + "completion_tokens": 2665, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-501", + "ok": false, + "error": null, + "completion_tokens": 5650, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-4", + "ok": true, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 618, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-444", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-228", + "ok": true, + "error": null, + "completion_tokens": 459, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-299", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-278", + "ok": true, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-154", + "ok": true, + "error": null, + "completion_tokens": 589, + "reasoning_chars": 1997, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-301", + "ok": true, + "error": null, + "completion_tokens": 378, + "reasoning_chars": 1138, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-161", + "ok": true, + "error": null, + "completion_tokens": 2724, + "reasoning_chars": 6615, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-394", + "ok": false, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-159", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 1055, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + } + ] + } + } + }, + "original-gguf": { + "started": "2026-10-10 15:40:38", + "finished": "2026-10-10 18:50:07", + "seed": 20260816, + "thinking": true, + "tasks": { + "mmlu": { + "task": "mmlu", + "n": 150, + "hits": 102, + "accuracy_pct": 68.0, + "accuracy_ex_trunc_pct": 68.0, + "truncated": 0, + "completion_tokens_p50": 53, + "completion_tokens_p90": 672, + "completion_tokens_max": 10397, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 1508, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 7859.5, + "gen_tokens": 52980, + "tok_per_s": 6.7, + "details": [ + { + "id": "mmlu-12325", + "ok": true, + "error": null, + "completion_tokens": 1041, + "reasoning_chars": 2458, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-154", + "ok": false, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 515, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12844", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1153", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11089", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12782", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5422", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 713, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12363", + "ok": true, + "error": null, + "completion_tokens": 2776, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12556", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13163", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-270", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6943", + "ok": true, + "error": null, + "completion_tokens": 472, + "reasoning_chars": 2067, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8330", + "ok": true, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 796, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9276", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7584", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 366, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12819", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6163", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 224, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4099", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 777, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10389", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 325, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1539", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12013", + "ok": true, + "error": null, + "completion_tokens": 798, + "reasoning_chars": 2459, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7814", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1106", + "ok": true, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1616", + "ok": true, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 560, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-555", + "ok": true, + "error": null, + "completion_tokens": 395, + "reasoning_chars": 1078, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3698", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-618", + "ok": false, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9842", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 508, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4229", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 121, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1022", + "ok": true, + "error": null, + "completion_tokens": 300, + "reasoning_chars": 977, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6421", + "ok": false, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1997", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 756, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8941", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-565", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11364", + "ok": false, + "error": null, + "completion_tokens": 10397, + "reasoning_chars": 39675, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5959", + "ok": true, + "error": null, + "completion_tokens": 433, + "reasoning_chars": 1355, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13066", + "ok": false, + "error": null, + "completion_tokens": 398, + "reasoning_chars": 1528, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3122", + "ok": true, + "error": null, + "completion_tokens": 509, + "reasoning_chars": 910, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3722", + "ok": true, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 1508, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13839", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12570", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9959", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 145, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10791", + "ok": false, + "error": null, + "completion_tokens": 1049, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12268", + "ok": true, + "error": null, + "completion_tokens": 888, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9051", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12887", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2667", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3906", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10304", + "ok": true, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12927", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6337", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 382, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3907", + "ok": true, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 668, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12888", + "ok": false, + "error": null, + "completion_tokens": 586, + "reasoning_chars": 2247, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10629", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 462, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5801", + "ok": true, + "error": null, + "completion_tokens": 378, + "reasoning_chars": 1160, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2936", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7389", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9532", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8705", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 286, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11776", + "ok": true, + "error": null, + "completion_tokens": 1763, + "reasoning_chars": 5868, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6836", + "ok": false, + "error": null, + "completion_tokens": 3109, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7856", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 633, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9697", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10825", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11178", + "ok": true, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11420", + "ok": true, + "error": null, + "completion_tokens": 698, + "reasoning_chars": 2098, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4658", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 893, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5403", + "ok": false, + "error": null, + "completion_tokens": 672, + "reasoning_chars": 2364, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6160", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6872", + "ok": false, + "error": null, + "completion_tokens": 7933, + "reasoning_chars": 32241, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9738", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 172, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3076", + "ok": true, + "error": null, + "completion_tokens": 443, + "reasoning_chars": 1296, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13496", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11289", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 598, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-222", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 145, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12180", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1143", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13086", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7038", + "ok": false, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13587", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12349", + "ok": true, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 720, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11747", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 1084, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9848", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 730, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3438", + "ok": true, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 796, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-89", + "ok": true, + "error": null, + "completion_tokens": 199, + "reasoning_chars": 808, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4253", + "ok": true, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 87, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5460", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2494", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11149", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-718", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5330", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10716", + "ok": true, + "error": null, + "completion_tokens": 1245, + "reasoning_chars": 3862, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5636", + "ok": true, + "error": null, + "completion_tokens": 691, + "reasoning_chars": 1268, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11832", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-95", + "ok": true, + "error": null, + "completion_tokens": 345, + "reasoning_chars": 998, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13960", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 237, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2072", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 17, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12030", + "ok": true, + "error": null, + "completion_tokens": 1477, + "reasoning_chars": 5394, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2931", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 1015, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2441", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13666", + "ok": true, + "error": null, + "completion_tokens": 344, + "reasoning_chars": 1383, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6575", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 751, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11625", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13283", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4262", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12931", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12150", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3678", + "ok": true, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4593", + "ok": true, + "error": null, + "completion_tokens": 366, + "reasoning_chars": 1136, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1563", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13396", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8049", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 676, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8396", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 902, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8493", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13514", + "ok": true, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 279, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3000", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 111, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1635", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 497, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1391", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-627", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1573", + "ok": true, + "error": null, + "completion_tokens": 414, + "reasoning_chars": 853, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6731", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11665", + "ok": true, + "error": null, + "completion_tokens": 815, + "reasoning_chars": 2355, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2914", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6875", + "ok": true, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 1344, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2184", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 735, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12801", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6717", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11", + "ok": true, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12464", + "ok": false, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2787", + "ok": false, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11549", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-842", + "ok": true, + "error": null, + "completion_tokens": 514, + "reasoning_chars": 1791, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13868", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9647", + "ok": true, + "error": null, + "completion_tokens": 370, + "reasoning_chars": 1430, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9502", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12441", + "ok": true, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 1358, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4392", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5036", + "ok": true, + "error": null, + "completion_tokens": 166, + "reasoning_chars": 737, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12606", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1400", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 335, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13979", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 712, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13634", + "ok": true, + "error": null, + "completion_tokens": 318, + "reasoning_chars": 1092, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11226", + "ok": true, + "error": null, + "completion_tokens": 1435, + "reasoning_chars": 4633, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-625", + "ok": false, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-257", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12853", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 583, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-451", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2074", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 42, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9117", + "ok": true, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 493, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10748", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "cmmlu": { + "task": "cmmlu", + "n": 150, + "hits": 92, + "accuracy_pct": 61.33, + "accuracy_ex_trunc_pct": 61.33, + "truncated": 0, + "completion_tokens_p50": 14, + "completion_tokens_p90": 292, + "completion_tokens_max": 1400, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 515, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 7772.0, + "gen_tokens": 16569, + "tok_per_s": 2.1, + "details": [ + { + "id": "cmmlu-chinese_teacher_qualification-63", + "ok": true, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 303, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-54", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 254, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-91", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-260", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-32", + "ok": true, + "error": null, + "completion_tokens": 48, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-29", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-200", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-98", + "ok": true, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 69, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-44", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 208, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-118", + "ok": true, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 774, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-116", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-43", + "ok": true, + "error": null, + "completion_tokens": 364, + "reasoning_chars": 515, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-145", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-247", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 339, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-224", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 299, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-26", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-105", + "ok": false, + "error": null, + "completion_tokens": 461, + "reasoning_chars": 690, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-15", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-62", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-nutrition-74", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-181", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 159, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-127", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_geography-107", + "ok": false, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 288, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-18", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-82", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_food_culture-38", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 150, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-198", + "ok": true, + "error": null, + "completion_tokens": 815, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-96", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 89, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-73", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-20", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-95", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-88", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-192", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-66", + "ok": true, + "error": null, + "completion_tokens": 248, + "reasoning_chars": 184, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-100", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-136", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-16", + "ok": false, + "error": null, + "completion_tokens": 298, + "reasoning_chars": 643, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-90", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 1287, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-41", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-345", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-103", + "ok": false, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 222, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-93", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-88", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-206", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-105", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-90", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-190", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-international_law-39", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-155", + "ok": true, + "error": null, + "completion_tokens": 1369, + "reasoning_chars": 4689, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-24", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-6", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-31", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-243", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-63", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-135", + "ok": true, + "error": null, + "completion_tokens": 54, + "reasoning_chars": 63, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-89", + "ok": false, + "error": null, + "completion_tokens": 72, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-91", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-120", + "ok": false, + "error": null, + "completion_tokens": 375, + "reasoning_chars": 769, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-62", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-106", + "ok": true, + "error": null, + "completion_tokens": 95, + "reasoning_chars": 150, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-food_science-106", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-24", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-157", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-41", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-115", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-journalism-29", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 110, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-67", + "ok": true, + "error": null, + "completion_tokens": 69, + "reasoning_chars": 91, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-161", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-120", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-education-117", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 220, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-169", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 217, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-129", + "ok": true, + "error": null, + "completion_tokens": 175, + "reasoning_chars": 271, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-19", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-8", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-72", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 571, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-public_relations-139", + "ok": false, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 372, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_security-22", + "ok": false, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-7", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-108", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-31", + "ok": true, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 29, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-131", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-35", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_actuarial_science-38", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-165", + "ok": true, + "error": null, + "completion_tokens": 64, + "reasoning_chars": 88, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-101", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-107", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 128, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-8", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medical_statistics-86", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-security_study-30", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 63, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-118", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_law-24", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-46", + "ok": true, + "error": null, + "completion_tokens": 689, + "reasoning_chars": 2272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-anatomy-21", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-145", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-arts-91", + "ok": true, + "error": null, + "completion_tokens": 156, + "reasoning_chars": 204, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-109", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 9, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-44", + "ok": false, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-74", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-63", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-astronomy-146", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-109", + "ok": true, + "error": null, + "completion_tokens": 402, + "reasoning_chars": 786, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-construction_project_management-16", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-113", + "ok": false, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 307, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-102", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-207", + "ok": false, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 262, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-19", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-164", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 305, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-338", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 55, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-7", + "ok": true, + "error": null, + "completion_tokens": 599, + "reasoning_chars": 1797, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-92", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-47", + "ok": true, + "error": null, + "completion_tokens": 339, + "reasoning_chars": 1131, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-astronomy-140", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-100", + "ok": false, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-37", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-191", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-34", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-88", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-public_relations-77", + "ok": false, + "error": null, + "completion_tokens": 404, + "reasoning_chars": 1212, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-306", + "ok": true, + "error": null, + "completion_tokens": 892, + "reasoning_chars": 2543, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-40", + "ok": true, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 68, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_chemistry-7", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-152", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 236, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-1", + "ok": true, + "error": null, + "completion_tokens": 119, + "reasoning_chars": 170, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-94", + "ok": true, + "error": null, + "completion_tokens": 93, + "reasoning_chars": 135, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-109", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 356, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-164", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 169, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-89", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-17", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-arts-2", + "ok": true, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 175, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-51", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-141", + "ok": true, + "error": null, + "completion_tokens": 258, + "reasoning_chars": 417, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-196", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-97", + "ok": true, + "error": null, + "completion_tokens": 191, + "reasoning_chars": 246, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-202", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-195", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-21", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-49", + "ok": true, + "error": null, + "completion_tokens": 1400, + "reasoning_chars": 2633, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-94", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-8", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-98", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 178, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-28", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-74", + "ok": false, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-121", + "ok": true, + "error": null, + "completion_tokens": 437, + "reasoning_chars": 813, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-120", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-105", + "ok": true, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 123, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-35", + "ok": true, + "error": null, + "completion_tokens": 222, + "reasoning_chars": 276, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-51", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-49", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-130", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-140", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "mmlu_pro": { + "task": "mmlu_pro", + "n": 150, + "hits": 96, + "accuracy_pct": 64.0, + "accuracy_ex_trunc_pct": 64.43, + "truncated": 1, + "completion_tokens_p50": 61, + "completion_tokens_p90": 1300, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 2971, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 7964.7, + "gen_tokens": 121349, + "tok_per_s": 15.2, + "details": [ + { + "id": "mmlupro-5831", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2299", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9280", + "ok": false, + "error": null, + "completion_tokens": 8540, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5486", + "ok": false, + "error": null, + "completion_tokens": 500, + "reasoning_chars": 2138, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5761", + "ok": true, + "error": null, + "completion_tokens": 88, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6589", + "ok": true, + "error": null, + "completion_tokens": 876, + "reasoning_chars": 2114, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5389", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5131", + "ok": false, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1677", + "ok": true, + "error": null, + "completion_tokens": 1373, + "reasoning_chars": 4868, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4388", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3285", + "ok": true, + "error": null, + "completion_tokens": 578, + "reasoning_chars": 1608, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4118", + "ok": true, + "error": null, + "completion_tokens": 739, + "reasoning_chars": 1816, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5344", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11696", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-858", + "ok": true, + "error": null, + "completion_tokens": 114, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-214", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6609", + "ok": true, + "error": null, + "completion_tokens": 991, + "reasoning_chars": 2835, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2083", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9752", + "ok": false, + "error": null, + "completion_tokens": 671, + "reasoning_chars": 1613, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7213", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5234", + "ok": false, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8570", + "ok": false, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5612", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2071", + "ok": true, + "error": null, + "completion_tokens": 597, + "reasoning_chars": 2000, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9842", + "ok": true, + "error": null, + "completion_tokens": 80, + "reasoning_chars": 100, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4665", + "ok": true, + "error": null, + "completion_tokens": 1026, + "reasoning_chars": 2282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5564", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6475", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 738, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2153", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5812", + "ok": true, + "error": null, + "completion_tokens": 166, + "reasoning_chars": 487, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3407", + "ok": false, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2879", + "ok": false, + "error": null, + "completion_tokens": 910, + "reasoning_chars": 1851, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8266", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 286, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1916", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3486", + "ok": false, + "error": null, + "completion_tokens": 38, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11275", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6040", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8565", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 256, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6848", + "ok": false, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8926", + "ok": true, + "error": null, + "completion_tokens": 277, + "reasoning_chars": 344, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8458", + "ok": true, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 88, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7854", + "ok": true, + "error": null, + "completion_tokens": 375, + "reasoning_chars": 964, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3962", + "ok": true, + "error": null, + "completion_tokens": 1980, + "reasoning_chars": 1928, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6470", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3813", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 763, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8026", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11030", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8151", + "ok": true, + "error": null, + "completion_tokens": 62, + "reasoning_chars": 81, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5286", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3888", + "ok": true, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 119, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11676", + "ok": true, + "error": null, + "completion_tokens": 1768, + "reasoning_chars": 3608, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8762", + "ok": false, + "error": null, + "completion_tokens": 973, + "reasoning_chars": 1949, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-12160", + "ok": true, + "error": null, + "completion_tokens": 2014, + "reasoning_chars": 4691, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4022", + "ok": true, + "error": null, + "completion_tokens": 1299, + "reasoning_chars": 3196, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10092", + "ok": false, + "error": null, + "completion_tokens": 6003, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7364", + "ok": true, + "error": null, + "completion_tokens": 350, + "reasoning_chars": 1442, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6001", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10873", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9777", + "ok": false, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8723", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4595", + "ok": true, + "error": null, + "completion_tokens": 356, + "reasoning_chars": 872, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8423", + "ok": true, + "error": null, + "completion_tokens": 7326, + "reasoning_chars": 10052, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11051", + "ok": false, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 789, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-836", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11297", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 82902, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "mmlupro-3950", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1562", + "ok": false, + "error": null, + "completion_tokens": 779, + "reasoning_chars": 2746, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2041", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3473", + "ok": true, + "error": null, + "completion_tokens": 346, + "reasoning_chars": 1528, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11253", + "ok": true, + "error": null, + "completion_tokens": 828, + "reasoning_chars": 2971, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3858", + "ok": true, + "error": null, + "completion_tokens": 479, + "reasoning_chars": 847, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2231", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2190", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6930", + "ok": true, + "error": null, + "completion_tokens": 471, + "reasoning_chars": 1804, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6558", + "ok": true, + "error": null, + "completion_tokens": 824, + "reasoning_chars": 2267, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5371", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1766", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6045", + "ok": true, + "error": null, + "completion_tokens": 1006, + "reasoning_chars": 3100, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2298", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6400", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6499", + "ok": true, + "error": null, + "completion_tokens": 98, + "reasoning_chars": 378, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7936", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 1058, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8566", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11633", + "ok": false, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9843", + "ok": false, + "error": null, + "completion_tokens": 7898, + "reasoning_chars": 19417, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11758", + "ok": true, + "error": null, + "completion_tokens": 9174, + "reasoning_chars": 16853, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2843", + "ok": true, + "error": null, + "completion_tokens": 1313, + "reasoning_chars": 3768, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7483", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11379", + "ok": false, + "error": null, + "completion_tokens": 51, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4866", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-818", + "ok": false, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2785", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8119", + "ok": true, + "error": null, + "completion_tokens": 424, + "reasoning_chars": 1293, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2214", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5774", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6049", + "ok": false, + "error": null, + "completion_tokens": 415, + "reasoning_chars": 1169, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7219", + "ok": true, + "error": null, + "completion_tokens": 492, + "reasoning_chars": 1788, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1660", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4767", + "ok": true, + "error": null, + "completion_tokens": 1300, + "reasoning_chars": 5100, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5964", + "ok": false, + "error": null, + "completion_tokens": 398, + "reasoning_chars": 944, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2743", + "ok": true, + "error": null, + "completion_tokens": 753, + "reasoning_chars": 2620, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4062", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 704, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7368", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-827", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11877", + "ok": false, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7557", + "ok": true, + "error": null, + "completion_tokens": 886, + "reasoning_chars": 3082, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9998", + "ok": true, + "error": null, + "completion_tokens": 218, + "reasoning_chars": 334, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10098", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 880, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1744", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9210", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 328, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11248", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 744, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11187", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2799", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4479", + "ok": true, + "error": null, + "completion_tokens": 1518, + "reasoning_chars": 3302, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10223", + "ok": true, + "error": null, + "completion_tokens": 540, + "reasoning_chars": 1166, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5698", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8291", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 416, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3516", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5466", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4851", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-733", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6995", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 188, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2573", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8481", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5938", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10708", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8476", + "ok": false, + "error": null, + "completion_tokens": 56, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7990", + "ok": true, + "error": null, + "completion_tokens": 293, + "reasoning_chars": 794, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2409", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11551", + "ok": false, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-674", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1978", + "ok": true, + "error": null, + "completion_tokens": 769, + "reasoning_chars": 2348, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10420", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3445", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9488", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 571, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4993", + "ok": true, + "error": null, + "completion_tokens": 501, + "reasoning_chars": 1746, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4316", + "ok": true, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 29, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4730", + "ok": true, + "error": null, + "completion_tokens": 411, + "reasoning_chars": 1328, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11570", + "ok": false, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2149", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8322", + "ok": true, + "error": null, + "completion_tokens": 1187, + "reasoning_chars": 2275, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10993", + "ok": false, + "error": null, + "completion_tokens": 794, + "reasoning_chars": 2418, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4588", + "ok": false, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3258", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8629", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-637", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 89, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4257", + "ok": true, + "error": null, + "completion_tokens": 427, + "reasoning_chars": 931, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9719", + "ok": true, + "error": null, + "completion_tokens": 1810, + "reasoning_chars": 4454, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11714", + "ok": false, + "error": null, + "completion_tokens": 6315, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9678", + "ok": true, + "error": null, + "completion_tokens": 1470, + "reasoning_chars": 5351, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "ceval": { + "task": "ceval", + "n": 150, + "hits": 90, + "accuracy_pct": 60.0, + "accuracy_ex_trunc_pct": 60.0, + "truncated": 0, + "completion_tokens_p50": 12, + "completion_tokens_p90": 448, + "completion_tokens_max": 31502, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 1074, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 3097.4, + "gen_tokens": 54577, + "tok_per_s": 17.6, + "details": [ + { + "id": "ceval-high_school_chinese-1", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-10", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-41", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_physics-3", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-21", + "ok": true, + "error": null, + "completion_tokens": 371, + "reasoning_chars": 563, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-4", + "ok": true, + "error": null, + "completion_tokens": 1904, + "reasoning_chars": 3988, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-21", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-2", + "ok": true, + "error": null, + "completion_tokens": 80, + "reasoning_chars": 124, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-6", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-12", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-19", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_chemistry-9", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-15", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-22", + "ok": false, + "error": null, + "completion_tokens": 101, + "reasoning_chars": 184, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-16", + "ok": true, + "error": null, + "completion_tokens": 38, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-11", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-18", + "ok": true, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 161, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-24", + "ok": true, + "error": null, + "completion_tokens": 531, + "reasoning_chars": 1605, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-29", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-2", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-11", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-18", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-0", + "ok": false, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 52, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-14", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-27", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-art_studies-19", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 149, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-21", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-33", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 348, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-20", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-10", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-19", + "ok": true, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_network-7", + "ok": true, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 58, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-41", + "ok": true, + "error": null, + "completion_tokens": 617, + "reasoning_chars": 1972, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-44", + "ok": true, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 79, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-9", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-28", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-39", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-1", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-24", + "ok": true, + "error": null, + "completion_tokens": 156, + "reasoning_chars": 272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-5", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_architecture-15", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-39", + "ok": true, + "error": null, + "completion_tokens": 386, + "reasoning_chars": 1756, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-23", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-7", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-17", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 1171, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_chemistry-6", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-15", + "ok": true, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 119, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-4", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-5", + "ok": true, + "error": null, + "completion_tokens": 1104, + "reasoning_chars": 3434, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-50", + "ok": true, + "error": null, + "completion_tokens": 366, + "reasoning_chars": 257, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_politics-11", + "ok": false, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-32", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-16", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-0", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-sports_science-9", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-art_studies-15", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-2", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-7", + "ok": true, + "error": null, + "completion_tokens": 256, + "reasoning_chars": 381, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-11", + "ok": true, + "error": null, + "completion_tokens": 166, + "reasoning_chars": 230, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-operating_system-18", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 135, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-4", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-17", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-1", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 301, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-32", + "ok": true, + "error": null, + "completion_tokens": 86, + "reasoning_chars": 203, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-33", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-25", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-40", + "ok": true, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 90, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-7", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-5", + "ok": true, + "error": null, + "completion_tokens": 398, + "reasoning_chars": 1889, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-12", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-4", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 169, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-38", + "ok": false, + "error": null, + "completion_tokens": 31502, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-22", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-0", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-15", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-45", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 101, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-6", + "ok": false, + "error": null, + "completion_tokens": 40, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-20", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-basic_medicine-2", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-17", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-16", + "ok": false, + "error": null, + "completion_tokens": 620, + "reasoning_chars": 2508, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-7", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_biology-16", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-10", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-32", + "ok": false, + "error": null, + "completion_tokens": 2280, + "reasoning_chars": 7445, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-0", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-12", + "ok": false, + "error": null, + "completion_tokens": 1735, + "reasoning_chars": 2714, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-14", + "ok": true, + "error": null, + "completion_tokens": 672, + "reasoning_chars": 2546, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-26", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-3", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-3", + "ok": true, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 43, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_network-13", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-sports_science-5", + "ok": true, + "error": null, + "completion_tokens": 448, + "reasoning_chars": 369, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-27", + "ok": true, + "error": null, + "completion_tokens": 2603, + "reasoning_chars": 3916, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-21", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-10", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 41, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-16", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-49", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-13", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-5", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-2", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 93, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-27", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_biology-8", + "ok": true, + "error": null, + "completion_tokens": 95, + "reasoning_chars": 126, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_physics-14", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-14", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-31", + "ok": true, + "error": null, + "completion_tokens": 459, + "reasoning_chars": 836, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-basic_medicine-13", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-13", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-12", + "ok": true, + "error": null, + "completion_tokens": 589, + "reasoning_chars": 1582, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-44", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-3", + "ok": true, + "error": null, + "completion_tokens": 1213, + "reasoning_chars": 3337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-13", + "ok": true, + "error": null, + "completion_tokens": 1089, + "reasoning_chars": 1074, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_mathematics-6", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_politics-19", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-environmental_impact_assessment_engineer-27", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-10", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 188, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-0", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-3", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 212, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-6", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_chinese-13", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-43", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-42", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-13", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-discrete_mathematics-1", + "ok": true, + "error": null, + "completion_tokens": 38, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-11", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 101, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-2", + "ok": true, + "error": null, + "completion_tokens": 90, + "reasoning_chars": 139, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-1", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_physics-16", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-11", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 218, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-5", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 281, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-40", + "ok": true, + "error": null, + "completion_tokens": 486, + "reasoning_chars": 1428, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-6", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 211, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_mathematics-10", + "ok": false, + "error": null, + "completion_tokens": 72, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-10", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-17", + "ok": false, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 219, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-11", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-16", + "ok": true, + "error": null, + "completion_tokens": 65, + "reasoning_chars": 89, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-1", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_physics-0", + "ok": false, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-37", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-9", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 96, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 338, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-5", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-18", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-30", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 169, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-21", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-42", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-5", + "ok": false, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_chemistry-4", + "ok": true, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "arc": { + "task": "arc", + "n": 150, + "hits": 135, + "accuracy_pct": 90.0, + "accuracy_ex_trunc_pct": 90.0, + "truncated": 0, + "completion_tokens_p50": 225, + "completion_tokens_p90": 443, + "completion_tokens_max": 1572, + "reasoning_chars_p50": 777, + "reasoning_chars_p90": 1302, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 7769.6, + "gen_tokens": 37617, + "tok_per_s": 4.8, + "details": [ + { + "id": "arc-Mercury_415084", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 48, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7249708", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 358, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_4_24", + "ok": true, + "error": null, + "completion_tokens": 242, + "reasoning_chars": 905, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7056823", + "ok": true, + "error": null, + "completion_tokens": 268, + "reasoning_chars": 755, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415534", + "ok": true, + "error": null, + "completion_tokens": 200, + "reasoning_chars": 525, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405827", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7263183", + "ok": true, + "error": null, + "completion_tokens": 261, + "reasoning_chars": 712, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7106960", + "ok": true, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 256, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_7", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 233, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7182245", + "ok": true, + "error": null, + "completion_tokens": 623, + "reasoning_chars": 1874, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416177", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-ACTAAP_2011_5_8", + "ok": true, + "error": null, + "completion_tokens": 341, + "reasoning_chars": 1041, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_8_36", + "ok": true, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 90, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401699", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 163, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7107240", + "ok": true, + "error": null, + "completion_tokens": 45, + "reasoning_chars": 207, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_4_26", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 728, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402638", + "ok": true, + "error": null, + "completion_tokens": 502, + "reasoning_chars": 1702, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7026723", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 273, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415422", + "ok": true, + "error": null, + "completion_tokens": 275, + "reasoning_chars": 961, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2000_8_2", + "ok": true, + "error": null, + "completion_tokens": 403, + "reasoning_chars": 1163, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2006_9_4", + "ok": true, + "error": null, + "completion_tokens": 562, + "reasoning_chars": 1660, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401272", + "ok": true, + "error": null, + "completion_tokens": 90, + "reasoning_chars": 391, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416169", + "ok": true, + "error": null, + "completion_tokens": 357, + "reasoning_chars": 799, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7116235", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 1325, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7109463", + "ok": true, + "error": null, + "completion_tokens": 250, + "reasoning_chars": 633, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7032428", + "ok": false, + "error": null, + "completion_tokens": 841, + "reasoning_chars": 2498, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AIMS_2008_8_11", + "ok": true, + "error": null, + "completion_tokens": 349, + "reasoning_chars": 1074, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_410334", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 1002, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_36", + "ok": true, + "error": null, + "completion_tokens": 90, + "reasoning_chars": 279, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415719", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 448, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_17", + "ok": true, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 807, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402256", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 547, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_400443", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7133245", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 781, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413458", + "ok": true, + "error": null, + "completion_tokens": 48, + "reasoning_chars": 182, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401372", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 714, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401116", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 359, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP__5_10312", + "ok": true, + "error": null, + "completion_tokens": 342, + "reasoning_chars": 911, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2007_4_10280", + "ok": true, + "error": null, + "completion_tokens": 187, + "reasoning_chars": 437, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_416683", + "ok": true, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 1500, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7190120", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 867, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401295", + "ok": true, + "error": null, + "completion_tokens": 362, + "reasoning_chars": 1439, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_4_28", + "ok": true, + "error": null, + "completion_tokens": 85, + "reasoning_chars": 347, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7223493", + "ok": true, + "error": null, + "completion_tokens": 265, + "reasoning_chars": 852, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7018410", + "ok": true, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 339, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1998_8_24", + "ok": true, + "error": null, + "completion_tokens": 762, + "reasoning_chars": 2272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_31", + "ok": true, + "error": null, + "completion_tokens": 316, + "reasoning_chars": 822, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405501", + "ok": true, + "error": null, + "completion_tokens": 297, + "reasoning_chars": 1012, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408030", + "ok": true, + "error": null, + "completion_tokens": 256, + "reasoning_chars": 842, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401684", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 690, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7041668", + "ok": true, + "error": null, + "completion_tokens": 403, + "reasoning_chars": 1292, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7205870", + "ok": true, + "error": null, + "completion_tokens": 300, + "reasoning_chars": 963, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7210140", + "ok": true, + "error": null, + "completion_tokens": 452, + "reasoning_chars": 1234, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_405873", + "ok": true, + "error": null, + "completion_tokens": 378, + "reasoning_chars": 1127, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400052", + "ok": true, + "error": null, + "completion_tokens": 211, + "reasoning_chars": 606, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7018270", + "ok": true, + "error": null, + "completion_tokens": 455, + "reasoning_chars": 1187, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_406550", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 810, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7150728", + "ok": true, + "error": null, + "completion_tokens": 514, + "reasoning_chars": 1289, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400839", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 710, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_30", + "ok": false, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 864, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400383", + "ok": true, + "error": null, + "completion_tokens": 479, + "reasoning_chars": 1302, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7236740", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-CSZ_2005_5_CSZ10021", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 883, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7093958", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 816, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7026355", + "ok": true, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 230, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408324", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 332, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP__7_10348", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 297, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402629", + "ok": true, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 190, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400046", + "ok": true, + "error": null, + "completion_tokens": 331, + "reasoning_chars": 1007, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7180810", + "ok": true, + "error": null, + "completion_tokens": 207, + "reasoning_chars": 1025, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_180005", + "ok": true, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 760, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AIMS_2009_8_14", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2007_5_16", + "ok": true, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 134, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_LBS10662", + "ok": true, + "error": null, + "completion_tokens": 133, + "reasoning_chars": 398, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_23", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 811, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2012_8_19", + "ok": false, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 524, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7041650", + "ok": true, + "error": null, + "completion_tokens": 359, + "reasoning_chars": 818, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416181", + "ok": true, + "error": null, + "completion_tokens": 80, + "reasoning_chars": 325, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TIMSS_1995_8_J6", + "ok": true, + "error": null, + "completion_tokens": 307, + "reasoning_chars": 998, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2014_5_13", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 526, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7106750", + "ok": true, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 962, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1999_4_23", + "ok": true, + "error": null, + "completion_tokens": 498, + "reasoning_chars": 1511, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MEA_2013_5_12", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 195, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_8_42", + "ok": false, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 427, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408367", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 368, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7250058", + "ok": true, + "error": null, + "completion_tokens": 350, + "reasoning_chars": 979, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_414041", + "ok": true, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 674, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401290", + "ok": true, + "error": null, + "completion_tokens": 186, + "reasoning_chars": 832, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_409295", + "ok": true, + "error": null, + "completion_tokens": 310, + "reasoning_chars": 1016, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7211680", + "ok": true, + "error": null, + "completion_tokens": 386, + "reasoning_chars": 1199, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_37", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 898, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413439", + "ok": true, + "error": null, + "completion_tokens": 413, + "reasoning_chars": 1211, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_32", + "ok": true, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 777, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7216423", + "ok": true, + "error": null, + "completion_tokens": 225, + "reasoning_chars": 1106, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-ACTAAP_2015_7_9", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7230405", + "ok": true, + "error": null, + "completion_tokens": 514, + "reasoning_chars": 1477, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7025008", + "ok": true, + "error": null, + "completion_tokens": 483, + "reasoning_chars": 1319, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_416536", + "ok": true, + "error": null, + "completion_tokens": 261, + "reasoning_chars": 760, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TIMSS_2007_8_pg60", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2009_4_34", + "ok": true, + "error": null, + "completion_tokens": 439, + "reasoning_chars": 1287, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_406833", + "ok": true, + "error": null, + "completion_tokens": 390, + "reasoning_chars": 1014, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_409647", + "ok": true, + "error": null, + "completion_tokens": 443, + "reasoning_chars": 1202, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_184363", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 814, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7107170", + "ok": true, + "error": null, + "completion_tokens": 286, + "reasoning_chars": 722, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416104", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 167, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_10-v1", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 176, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7202843", + "ok": true, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 316, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7029803", + "ok": true, + "error": null, + "completion_tokens": 360, + "reasoning_chars": 1175, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7164623", + "ok": true, + "error": null, + "completion_tokens": 271, + "reasoning_chars": 896, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_26", + "ok": true, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 222, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7263060", + "ok": true, + "error": null, + "completion_tokens": 390, + "reasoning_chars": 1003, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7268328", + "ok": true, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 212, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402282", + "ok": true, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 290, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7029558", + "ok": true, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 205, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_4_21", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 397, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7186358", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 1084, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2011_8_10434", + "ok": true, + "error": null, + "completion_tokens": 420, + "reasoning_chars": 1166, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2012_8_23651", + "ok": true, + "error": null, + "completion_tokens": 246, + "reasoning_chars": 624, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_407125", + "ok": true, + "error": null, + "completion_tokens": 1572, + "reasoning_chars": 5303, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7268258", + "ok": false, + "error": null, + "completion_tokens": 269, + "reasoning_chars": 860, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7027405", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 594, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_8_7", + "ok": true, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 1086, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_191625", + "ok": false, + "error": null, + "completion_tokens": 718, + "reasoning_chars": 3139, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_6", + "ok": true, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 177, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7219678", + "ok": true, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 1108, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401678", + "ok": true, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 165, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401312", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_8_42", + "ok": false, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 167, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7030468", + "ok": true, + "error": null, + "completion_tokens": 400, + "reasoning_chars": 1114, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_175385", + "ok": true, + "error": null, + "completion_tokens": 402, + "reasoning_chars": 1063, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_175560", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_5", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 137, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_409009", + "ok": true, + "error": null, + "completion_tokens": 262, + "reasoning_chars": 966, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7128923", + "ok": false, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 1310, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_8_9", + "ok": false, + "error": null, + "completion_tokens": 170, + "reasoning_chars": 589, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7042945", + "ok": true, + "error": null, + "completion_tokens": 309, + "reasoning_chars": 781, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_8_47", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 646, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7185343", + "ok": true, + "error": null, + "completion_tokens": 536, + "reasoning_chars": 1523, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7218400", + "ok": true, + "error": null, + "completion_tokens": 340, + "reasoning_chars": 922, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7221988", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 534, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_405950", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 514, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7222530", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 971, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7102323", + "ok": true, + "error": null, + "completion_tokens": 131, + "reasoning_chars": 361, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2009_5_39", + "ok": true, + "error": null, + "completion_tokens": 247, + "reasoning_chars": 580, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1999_8_5", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 424, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400661", + "ok": true, + "error": null, + "completion_tokens": 380, + "reasoning_chars": 999, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7081603", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 890, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2008_3_25", + "ok": true, + "error": null, + "completion_tokens": 158, + "reasoning_chars": 420, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413628", + "ok": true, + "error": null, + "completion_tokens": 186, + "reasoning_chars": 392, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_406427", + "ok": true, + "error": null, + "completion_tokens": 433, + "reasoning_chars": 1029, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "truthfulqa": { + "task": "truthfulqa", + "n": 150, + "hits": 91, + "accuracy_pct": 60.67, + "accuracy_ex_trunc_pct": 60.67, + "truncated": 0, + "completion_tokens_p50": 6, + "completion_tokens_p90": 119, + "completion_tokens_max": 658, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 284, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 4535.0, + "gen_tokens": 6431, + "tok_per_s": 1.4, + "details": [ + { + "id": "tqa-210", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-76", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-461", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-218", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-254", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-635", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-193", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-148", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-400", + "ok": false, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 661, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-766", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-240", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-50", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-83", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-381", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-325", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-655", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-721", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-391", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-401", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-625", + "ok": false, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-80", + "ok": false, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 852, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-439", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-66", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-155", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-516", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-522", + "ok": true, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-47", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-343", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-126", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-77", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-613", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-590", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-674", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-438", + "ok": false, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 709, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-503", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-639", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-111", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-289", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-705", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-373", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-363", + "ok": true, + "error": null, + "completion_tokens": 529, + "reasoning_chars": 1831, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-792", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-113", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-74", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-634", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 682, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-804", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-626", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-676", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-174", + "ok": true, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-455", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-725", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-296", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-535", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-172", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-23", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-43", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-38", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-97", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-657", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-814", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-346", + "ok": true, + "error": null, + "completion_tokens": 264, + "reasoning_chars": 924, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-365", + "ok": true, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-384", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-677", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-486", + "ok": false, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-55", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-680", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-510", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-128", + "ok": true, + "error": null, + "completion_tokens": 386, + "reasoning_chars": 1300, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-805", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-394", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-435", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-285", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-687", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-588", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-100", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-399", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-266", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-693", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 246, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-532", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-558", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-319", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-459", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-675", + "ok": true, + "error": null, + "completion_tokens": 431, + "reasoning_chars": 1318, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-129", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-331", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-12", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-164", + "ok": false, + "error": null, + "completion_tokens": 582, + "reasoning_chars": 1580, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-306", + "ok": true, + "error": null, + "completion_tokens": 119, + "reasoning_chars": 284, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-275", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-418", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-264", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-94", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-389", + "ok": true, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-427", + "ok": true, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-269", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-643", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-7", + "ok": true, + "error": null, + "completion_tokens": 658, + "reasoning_chars": 1983, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-419", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-616", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-699", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-473", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-523", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-214", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 489, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-376", + "ok": true, + "error": null, + "completion_tokens": 56, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-123", + "ok": true, + "error": null, + "completion_tokens": 203, + "reasoning_chars": 785, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-200", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-493", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-474", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-387", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-746", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-790", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-181", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-413", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-349", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-434", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-601", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-489", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-286", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-488", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-759", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-273", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-59", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-315", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-425", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-58", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-691", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-497", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-515", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-398", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-670", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-170", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-334", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-307", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-495", + "ok": true, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 928, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-597", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-775", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-186", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 841, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-733", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-308", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-5", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-231", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 751, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-356", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-810", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-270", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-13", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-114", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-4", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-417", + "ok": true, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-750", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "gsm8k": { + "task": "gsm8k", + "n": 100, + "hits": 37, + "accuracy_pct": 37.0, + "accuracy_ex_trunc_pct": 37.0, + "truncated": 0, + "completion_tokens_p50": 7, + "completion_tokens_p90": 17, + "completion_tokens_max": 2152, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 0, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 5886.8, + "gen_tokens": 5333, + "tok_per_s": 0.9, + "details": [ + { + "id": "gsm8k-144", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-627", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-195", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-201", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-730", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-166", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1141", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-832", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-53", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1042", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-609", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-318", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-892", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-204", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1120", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1040", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-556", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1015", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-206", + "ok": true, + "error": null, + "completion_tokens": 282, + "reasoning_chars": 582, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1058", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1044", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-131", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1154", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-158", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1149", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1156", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-60", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-918", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1010", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1032", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1053", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-577", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-935", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-692", + "ok": true, + "error": null, + "completion_tokens": 259, + "reasoning_chars": 407, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-153", + "ok": true, + "error": null, + "completion_tokens": 698, + "reasoning_chars": 1693, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-40", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-613", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-875", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-189", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-800", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-169", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1169", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-211", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1183", + "ok": true, + "error": null, + "completion_tokens": 2152, + "reasoning_chars": 8585, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-193", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-230", + "ok": false, + "error": null, + "completion_tokens": 71, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-787", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-83", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1318", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-733", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1205", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-88", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-509", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-311", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-874", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-287", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 449, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1055", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1246", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-975", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-280", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-541", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-974", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-827", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-125", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-453", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-942", + "ok": true, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1199", + "ok": false, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-884", + "ok": true, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-855", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1270", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-349", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1231", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1106", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-803", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-922", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-152", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-920", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-207", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-297", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-786", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-101", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1301", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-27", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1005", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-659", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-340", + "ok": true, + "error": null, + "completion_tokens": 876, + "reasoning_chars": 3280, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-288", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-763", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-840", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-742", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-222", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-468", + "ok": true, + "error": null, + "completion_tokens": 45, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-220", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-662", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-269", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1193", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-270", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-114", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-728", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-396", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "math500": { + "task": "math500", + "n": 100, + "hits": 68, + "accuracy_pct": 68.0, + "accuracy_ex_trunc_pct": 68.0, + "truncated": 0, + "completion_tokens_p50": 364, + "completion_tokens_p90": 3353, + "completion_tokens_max": 23656, + "reasoning_chars_p50": 625, + "reasoning_chars_p90": 7342, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 10017.1, + "gen_tokens": 121453, + "tok_per_s": 12.1, + "details": [ + { + "id": "math-test/algebra/1553.json", + "ok": true, + "error": null, + "completion_tokens": 496, + "reasoning_chars": 750, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/538.json", + "ok": true, + "error": null, + "completion_tokens": 525, + "reasoning_chars": 661, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/691.json", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1090.json", + "ok": true, + "error": null, + "completion_tokens": 672, + "reasoning_chars": 1440, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/116.json", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 389, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1255.json", + "ok": true, + "error": null, + "completion_tokens": 624, + "reasoning_chars": 1837, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1143.json", + "ok": true, + "error": null, + "completion_tokens": 684, + "reasoning_chars": 895, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1201.json", + "ok": true, + "error": null, + "completion_tokens": 23656, + "reasoning_chars": 53668, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/964.json", + "ok": true, + "error": null, + "completion_tokens": 3512, + "reasoning_chars": 6056, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/662.json", + "ok": false, + "error": null, + "completion_tokens": 4064, + "reasoning_chars": 7342, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/153.json", + "ok": false, + "error": null, + "completion_tokens": 222, + "reasoning_chars": 346, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/276.json", + "ok": true, + "error": null, + "completion_tokens": 418, + "reasoning_chars": 611, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1840.json", + "ok": true, + "error": null, + "completion_tokens": 245, + "reasoning_chars": 410, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1072.json", + "ok": false, + "error": null, + "completion_tokens": 436, + "reasoning_chars": 618, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1907.json", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/567.json", + "ok": true, + "error": null, + "completion_tokens": 364, + "reasoning_chars": 533, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1922.json", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1779.json", + "ok": true, + "error": null, + "completion_tokens": 4705, + "reasoning_chars": 11788, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/951.json", + "ok": true, + "error": null, + "completion_tokens": 302, + "reasoning_chars": 444, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1197.json", + "ok": false, + "error": null, + "completion_tokens": 9506, + "reasoning_chars": 18657, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/853.json", + "ok": true, + "error": null, + "completion_tokens": 853, + "reasoning_chars": 1276, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/516.json", + "ok": false, + "error": null, + "completion_tokens": 1255, + "reasoning_chars": 3186, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1114.json", + "ok": false, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 374, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1065.json", + "ok": true, + "error": null, + "completion_tokens": 462, + "reasoning_chars": 598, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1232.json", + "ok": false, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 305, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/769.json", + "ok": true, + "error": null, + "completion_tokens": 558, + "reasoning_chars": 566, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1791.json", + "ok": false, + "error": null, + "completion_tokens": 1193, + "reasoning_chars": 2837, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/246.json", + "ok": true, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 293, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1166.json", + "ok": true, + "error": null, + "completion_tokens": 4882, + "reasoning_chars": 8653, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1410.json", + "ok": false, + "error": null, + "completion_tokens": 394, + "reasoning_chars": 606, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1686.json", + "ok": true, + "error": null, + "completion_tokens": 160, + "reasoning_chars": 334, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/754.json", + "ok": false, + "error": null, + "completion_tokens": 697, + "reasoning_chars": 1422, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/477.json", + "ok": true, + "error": null, + "completion_tokens": 392, + "reasoning_chars": 1349, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2680.json", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 340, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/23957.json", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1865.json", + "ok": true, + "error": null, + "completion_tokens": 604, + "reasoning_chars": 1337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1014.json", + "ok": true, + "error": null, + "completion_tokens": 338, + "reasoning_chars": 715, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/431.json", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 271, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1425.json", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 237, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2584.json", + "ok": false, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 260, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2470.json", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 50, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/547.json", + "ok": true, + "error": null, + "completion_tokens": 2677, + "reasoning_chars": 7832, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1105.json", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 503, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/347.json", + "ok": true, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 625, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1257.json", + "ok": true, + "error": null, + "completion_tokens": 429, + "reasoning_chars": 882, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1126.json", + "ok": true, + "error": null, + "completion_tokens": 529, + "reasoning_chars": 747, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/239.json", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1961.json", + "ok": false, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 817, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1388.json", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1078.json", + "ok": true, + "error": null, + "completion_tokens": 725, + "reasoning_chars": 1399, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2157.json", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 494, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/339.json", + "ok": true, + "error": null, + "completion_tokens": 1031, + "reasoning_chars": 3239, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1761.json", + "ok": true, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 260, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1960.json", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 220, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/2086.json", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1462.json", + "ok": true, + "error": null, + "completion_tokens": 6188, + "reasoning_chars": 11376, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1060.json", + "ok": true, + "error": null, + "completion_tokens": 798, + "reasoning_chars": 981, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1055.json", + "ok": true, + "error": null, + "completion_tokens": 2924, + "reasoning_chars": 8058, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2517.json", + "ok": false, + "error": null, + "completion_tokens": 486, + "reasoning_chars": 933, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/996.json", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/893.json", + "ok": true, + "error": null, + "completion_tokens": 465, + "reasoning_chars": 498, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/51.json", + "ok": false, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1291.json", + "ok": true, + "error": null, + "completion_tokens": 438, + "reasoning_chars": 670, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/694.json", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/711.json", + "ok": true, + "error": null, + "completion_tokens": 5909, + "reasoning_chars": 18936, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1655.json", + "ok": false, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 419, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1936.json", + "ok": true, + "error": null, + "completion_tokens": 144, + "reasoning_chars": 234, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1572.json", + "ok": true, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 690, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/509.json", + "ok": false, + "error": null, + "completion_tokens": 476, + "reasoning_chars": 763, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/864.json", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1405.json", + "ok": true, + "error": null, + "completion_tokens": 1680, + "reasoning_chars": 4006, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/192.json", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2102.json", + "ok": true, + "error": null, + "completion_tokens": 347, + "reasoning_chars": 519, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/843.json", + "ok": true, + "error": null, + "completion_tokens": 418, + "reasoning_chars": 686, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/187.json", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 433, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/612.json", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 1115, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/615.json", + "ok": true, + "error": null, + "completion_tokens": 1085, + "reasoning_chars": 3063, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1908.json", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/928.json", + "ok": true, + "error": null, + "completion_tokens": 995, + "reasoning_chars": 2230, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/505.json", + "ok": true, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1930.json", + "ok": false, + "error": null, + "completion_tokens": 3353, + "reasoning_chars": 6577, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1934.json", + "ok": true, + "error": null, + "completion_tokens": 272, + "reasoning_chars": 204, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1139.json", + "ok": true, + "error": null, + "completion_tokens": 9872, + "reasoning_chars": 28089, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1000.json", + "ok": true, + "error": null, + "completion_tokens": 2432, + "reasoning_chars": 6596, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/183.json", + "ok": true, + "error": null, + "completion_tokens": 486, + "reasoning_chars": 775, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1275.json", + "ok": true, + "error": null, + "completion_tokens": 571, + "reasoning_chars": 1080, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2700.json", + "ok": false, + "error": null, + "completion_tokens": 811, + "reasoning_chars": 2204, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2257.json", + "ok": false, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 424, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/14.json", + "ok": false, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 474, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1300.json", + "ok": false, + "error": null, + "completion_tokens": 736, + "reasoning_chars": 1070, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/515.json", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 625, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1423.json", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1102.json", + "ok": true, + "error": null, + "completion_tokens": 540, + "reasoning_chars": 996, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1646.json", + "ok": true, + "error": null, + "completion_tokens": 4379, + "reasoning_chars": 11633, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1272.json", + "ok": false, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 262, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/25149.json", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 445, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/697.json", + "ok": true, + "error": null, + "completion_tokens": 1951, + "reasoning_chars": 3601, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2036.json", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 299, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/2037.json", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/353.json", + "ok": true, + "error": null, + "completion_tokens": 308, + "reasoning_chars": 638, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "bbh": { + "task": "bbh", + "n": 150, + "hits": 72, + "accuracy_pct": 48.0, + "accuracy_ex_trunc_pct": 48.0, + "truncated": 0, + "completion_tokens_p50": 46, + "completion_tokens_p90": 576, + "completion_tokens_max": 10727, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 1589, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 7919.7, + "gen_tokens": 65087, + "tok_per_s": 8.2, + "details": [ + { + "id": "bbh-sports_understanding-101", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-60", + "ok": true, + "error": null, + "completion_tokens": 476, + "reasoning_chars": 1495, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-113", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-31", + "ok": true, + "error": null, + "completion_tokens": 676, + "reasoning_chars": 2396, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-144", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-28", + "ok": true, + "error": null, + "completion_tokens": 399, + "reasoning_chars": 583, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-116", + "ok": true, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 147, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-67", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-3", + "ok": true, + "error": null, + "completion_tokens": 262, + "reasoning_chars": 1142, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-29", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-192", + "ok": true, + "error": null, + "completion_tokens": 284, + "reasoning_chars": 1050, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-196", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-10", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-83", + "ok": true, + "error": null, + "completion_tokens": 394, + "reasoning_chars": 1035, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-76", + "ok": false, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-212", + "ok": true, + "error": null, + "completion_tokens": 398, + "reasoning_chars": 1174, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-87", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-1", + "ok": true, + "error": null, + "completion_tokens": 272, + "reasoning_chars": 924, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-140", + "ok": true, + "error": null, + "completion_tokens": 546, + "reasoning_chars": 2453, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-8", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 664, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-138", + "ok": true, + "error": null, + "completion_tokens": 72, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-221", + "ok": true, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-136", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-14", + "ok": false, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-0", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-207", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-32", + "ok": true, + "error": null, + "completion_tokens": 367, + "reasoning_chars": 1589, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-83", + "ok": true, + "error": null, + "completion_tokens": 406, + "reasoning_chars": 1135, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-220", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-234", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-61", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-21", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-31", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-13", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-211", + "ok": true, + "error": null, + "completion_tokens": 478, + "reasoning_chars": 1476, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-37", + "ok": true, + "error": null, + "completion_tokens": 217, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-225", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-90", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 54, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-101", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-179", + "ok": true, + "error": null, + "completion_tokens": 336, + "reasoning_chars": 1119, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-173", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 644, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-105", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-88", + "ok": false, + "error": null, + "completion_tokens": 65, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-68", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-185", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-227", + "ok": true, + "error": null, + "completion_tokens": 67, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-129", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-238", + "ok": true, + "error": null, + "completion_tokens": 266, + "reasoning_chars": 684, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-202", + "ok": true, + "error": null, + "completion_tokens": 314, + "reasoning_chars": 528, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-93", + "ok": true, + "error": null, + "completion_tokens": 230, + "reasoning_chars": 990, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-93", + "ok": false, + "error": null, + "completion_tokens": 1131, + "reasoning_chars": 4350, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-161", + "ok": false, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-46", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-227", + "ok": true, + "error": null, + "completion_tokens": 394, + "reasoning_chars": 1349, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-158", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-205", + "ok": true, + "error": null, + "completion_tokens": 451, + "reasoning_chars": 1079, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-89", + "ok": false, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-177", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-65", + "ok": true, + "error": null, + "completion_tokens": 262, + "reasoning_chars": 636, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-72", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-49", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 285, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-54", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-64", + "ok": false, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-86", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-108", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 173, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-194", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-194", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-75", + "ok": true, + "error": null, + "completion_tokens": 5290, + "reasoning_chars": 18374, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-106", + "ok": true, + "error": null, + "completion_tokens": 398, + "reasoning_chars": 1239, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-81", + "ok": false, + "error": null, + "completion_tokens": 5987, + "reasoning_chars": 27415, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-242", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-150", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 874, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-25", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-139", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-12", + "ok": true, + "error": null, + "completion_tokens": 346, + "reasoning_chars": 1277, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-178", + "ok": true, + "error": null, + "completion_tokens": 656, + "reasoning_chars": 2580, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-160", + "ok": false, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-115", + "ok": true, + "error": null, + "completion_tokens": 5216, + "reasoning_chars": 22632, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-44", + "ok": true, + "error": null, + "completion_tokens": 675, + "reasoning_chars": 2080, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-189", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 1365, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-102", + "ok": true, + "error": null, + "completion_tokens": 311, + "reasoning_chars": 1111, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-38", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-141", + "ok": false, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-38", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 90, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-168", + "ok": true, + "error": null, + "completion_tokens": 556, + "reasoning_chars": 1328, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-212", + "ok": false, + "error": null, + "completion_tokens": 10727, + "reasoning_chars": 26480, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-245", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 578, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-222", + "ok": true, + "error": null, + "completion_tokens": 282, + "reasoning_chars": 1049, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-244", + "ok": true, + "error": null, + "completion_tokens": 262, + "reasoning_chars": 916, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-35", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-126", + "ok": false, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-90", + "ok": false, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-5", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-241", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-234", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 930, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-233", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-5", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-152", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-9", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-209", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-171", + "ok": true, + "error": null, + "completion_tokens": 327, + "reasoning_chars": 946, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-160", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-166", + "ok": false, + "error": null, + "completion_tokens": 2634, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-213", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-240", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 77, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-95", + "ok": false, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-227", + "ok": true, + "error": null, + "completion_tokens": 496, + "reasoning_chars": 1627, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-96", + "ok": true, + "error": null, + "completion_tokens": 615, + "reasoning_chars": 2650, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-33", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-4", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-180", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-74", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 961, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-88", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-10", + "ok": true, + "error": null, + "completion_tokens": 221, + "reasoning_chars": 811, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-191", + "ok": false, + "error": null, + "completion_tokens": 3542, + "reasoning_chars": 14420, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-71", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-150", + "ok": false, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-188", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 251, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-191", + "ok": true, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 437, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-203", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 1090, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-237", + "ok": true, + "error": null, + "completion_tokens": 345, + "reasoning_chars": 687, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-24", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-27", + "ok": true, + "error": null, + "completion_tokens": 440, + "reasoning_chars": 1282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-177", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 973, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-192", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-242", + "ok": false, + "error": null, + "completion_tokens": 56, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-51", + "ok": false, + "error": null, + "completion_tokens": 7450, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-96", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-100", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-204", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-58", + "ok": true, + "error": null, + "completion_tokens": 576, + "reasoning_chars": 1762, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-157", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-104", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 621, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-47", + "ok": false, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-150", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-23", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-68", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-174", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 186, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-108", + "ok": true, + "error": null, + "completion_tokens": 1233, + "reasoning_chars": 4270, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-203", + "ok": false, + "error": null, + "completion_tokens": 402, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-115", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-223", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-221", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-5", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-233", + "ok": true, + "error": null, + "completion_tokens": 891, + "reasoning_chars": 628, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-3", + "ok": true, + "error": null, + "completion_tokens": 294, + "reasoning_chars": 1025, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-119", + "ok": true, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 518, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-197", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-103", + "ok": true, + "error": null, + "completion_tokens": 1031, + "reasoning_chars": 4489, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-178", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "humaneval": { + "task": "humaneval", + "n": 164, + "hits": 149, + "accuracy_pct": 90.85, + "accuracy_ex_trunc_pct": 91.41, + "truncated": 1, + "completion_tokens_p50": 488, + "completion_tokens_p90": 1470, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 1023, + "reasoning_chars_p90": 3191, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 11369.1, + "gen_tokens": 208941, + "tok_per_s": 18.4, + "details": [ + { + "id": "HumanEval/0", + "ok": true, + "error": null, + "completion_tokens": 667, + "reasoning_chars": 2166, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/1", + "ok": true, + "error": null, + "completion_tokens": 498, + "reasoning_chars": 1653, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/2", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/3", + "ok": true, + "error": null, + "completion_tokens": 314, + "reasoning_chars": 1264, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/4", + "ok": false, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/5", + "ok": false, + "error": null, + "completion_tokens": 653, + "reasoning_chars": 2358, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/6", + "ok": false, + "error": null, + "completion_tokens": 625, + "reasoning_chars": 2585, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/7", + "ok": true, + "error": null, + "completion_tokens": 296, + "reasoning_chars": 890, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/8", + "ok": false, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/9", + "ok": true, + "error": null, + "completion_tokens": 594, + "reasoning_chars": 2477, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/10", + "ok": false, + "error": null, + "completion_tokens": 866, + "reasoning_chars": 3064, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/11", + "ok": true, + "error": null, + "completion_tokens": 884, + "reasoning_chars": 2902, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/12", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/13", + "ok": true, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/14", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 746, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/15", + "ok": true, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/16", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/17", + "ok": true, + "error": null, + "completion_tokens": 615, + "reasoning_chars": 1819, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/18", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/19", + "ok": true, + "error": null, + "completion_tokens": 420, + "reasoning_chars": 1318, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/20", + "ok": false, + "error": null, + "completion_tokens": 771, + "reasoning_chars": 3061, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/21", + "ok": false, + "error": null, + "completion_tokens": 2190, + "reasoning_chars": 7709, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/22", + "ok": true, + "error": null, + "completion_tokens": 1829, + "reasoning_chars": 7699, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/23", + "ok": true, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/24", + "ok": true, + "error": null, + "completion_tokens": 853, + "reasoning_chars": 2618, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/25", + "ok": true, + "error": null, + "completion_tokens": 1349, + "reasoning_chars": 4383, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/26", + "ok": true, + "error": null, + "completion_tokens": 488, + "reasoning_chars": 1894, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/27", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/28", + "ok": true, + "error": null, + "completion_tokens": 252, + "reasoning_chars": 830, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/29", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 766, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/30", + "ok": true, + "error": null, + "completion_tokens": 234, + "reasoning_chars": 462, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/31", + "ok": true, + "error": null, + "completion_tokens": 456, + "reasoning_chars": 981, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/32", + "ok": false, + "error": null, + "completion_tokens": 9935, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/33", + "ok": true, + "error": null, + "completion_tokens": 519, + "reasoning_chars": 1023, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/34", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 598, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/35", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/36", + "ok": true, + "error": null, + "completion_tokens": 1061, + "reasoning_chars": 2437, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/37", + "ok": true, + "error": null, + "completion_tokens": 724, + "reasoning_chars": 1991, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/38", + "ok": false, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/39", + "ok": true, + "error": null, + "completion_tokens": 15036, + "reasoning_chars": 39762, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/40", + "ok": true, + "error": null, + "completion_tokens": 515, + "reasoning_chars": 1095, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/41", + "ok": true, + "error": null, + "completion_tokens": 404, + "reasoning_chars": 1089, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/42", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/43", + "ok": true, + "error": null, + "completion_tokens": 526, + "reasoning_chars": 1203, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/44", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 719, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/45", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/46", + "ok": true, + "error": null, + "completion_tokens": 692, + "reasoning_chars": 944, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/47", + "ok": true, + "error": null, + "completion_tokens": 69, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/48", + "ok": true, + "error": null, + "completion_tokens": 166, + "reasoning_chars": 397, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/49", + "ok": true, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/50", + "ok": false, + "error": null, + "completion_tokens": 273, + "reasoning_chars": 836, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/51", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/52", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/53", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/54", + "ok": true, + "error": null, + "completion_tokens": 348, + "reasoning_chars": 706, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/55", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/56", + "ok": true, + "error": null, + "completion_tokens": 328, + "reasoning_chars": 749, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/57", + "ok": true, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/58", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 660, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/59", + "ok": false, + "error": null, + "completion_tokens": 1121, + "reasoning_chars": 2891, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/60", + "ok": true, + "error": null, + "completion_tokens": 680, + "reasoning_chars": 2011, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/61", + "ok": true, + "error": null, + "completion_tokens": 286, + "reasoning_chars": 952, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/62", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 386, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/63", + "ok": true, + "error": null, + "completion_tokens": 770, + "reasoning_chars": 1560, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/64", + "ok": true, + "error": null, + "completion_tokens": 650, + "reasoning_chars": 1735, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/65", + "ok": true, + "error": null, + "completion_tokens": 1048, + "reasoning_chars": 3184, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/66", + "ok": true, + "error": null, + "completion_tokens": 1372, + "reasoning_chars": 4297, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/67", + "ok": true, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/68", + "ok": true, + "error": null, + "completion_tokens": 598, + "reasoning_chars": 758, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/69", + "ok": true, + "error": null, + "completion_tokens": 339, + "reasoning_chars": 1083, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/70", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 912, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/71", + "ok": true, + "error": null, + "completion_tokens": 88, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/72", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 508, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/73", + "ok": true, + "error": null, + "completion_tokens": 474, + "reasoning_chars": 932, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/74", + "ok": true, + "error": null, + "completion_tokens": 659, + "reasoning_chars": 1515, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/75", + "ok": true, + "error": null, + "completion_tokens": 3077, + "reasoning_chars": 9362, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/76", + "ok": true, + "error": null, + "completion_tokens": 7050, + "reasoning_chars": 20747, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/77", + "ok": true, + "error": null, + "completion_tokens": 1522, + "reasoning_chars": 3804, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/78", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/79", + "ok": true, + "error": null, + "completion_tokens": 343, + "reasoning_chars": 652, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/80", + "ok": true, + "error": null, + "completion_tokens": 76, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/81", + "ok": true, + "error": null, + "completion_tokens": 1079, + "reasoning_chars": 1501, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/82", + "ok": true, + "error": null, + "completion_tokens": 335, + "reasoning_chars": 774, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/83", + "ok": true, + "error": null, + "completion_tokens": 925, + "reasoning_chars": 2338, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/84", + "ok": true, + "error": null, + "completion_tokens": 315, + "reasoning_chars": 483, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/85", + "ok": true, + "error": null, + "completion_tokens": 273, + "reasoning_chars": 748, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/86", + "ok": true, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/87", + "ok": true, + "error": null, + "completion_tokens": 653, + "reasoning_chars": 1306, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/88", + "ok": true, + "error": null, + "completion_tokens": 566, + "reasoning_chars": 1171, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/89", + "ok": true, + "error": null, + "completion_tokens": 576, + "reasoning_chars": 1167, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/90", + "ok": true, + "error": null, + "completion_tokens": 391, + "reasoning_chars": 1025, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/91", + "ok": true, + "error": null, + "completion_tokens": 725, + "reasoning_chars": 2198, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/92", + "ok": true, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/93", + "ok": true, + "error": null, + "completion_tokens": 1177, + "reasoning_chars": 3928, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/94", + "ok": true, + "error": null, + "completion_tokens": 1470, + "reasoning_chars": 2876, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/95", + "ok": true, + "error": null, + "completion_tokens": 559, + "reasoning_chars": 1956, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/96", + "ok": true, + "error": null, + "completion_tokens": 994, + "reasoning_chars": 2446, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/97", + "ok": true, + "error": null, + "completion_tokens": 220, + "reasoning_chars": 624, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/98", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 635, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/99", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 42156, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "HumanEval/100", + "ok": true, + "error": null, + "completion_tokens": 619, + "reasoning_chars": 1609, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/101", + "ok": true, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/102", + "ok": true, + "error": null, + "completion_tokens": 3471, + "reasoning_chars": 10426, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/103", + "ok": true, + "error": null, + "completion_tokens": 839, + "reasoning_chars": 2107, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/104", + "ok": true, + "error": null, + "completion_tokens": 308, + "reasoning_chars": 649, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/105", + "ok": true, + "error": null, + "completion_tokens": 104, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/106", + "ok": true, + "error": null, + "completion_tokens": 657, + "reasoning_chars": 1454, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/107", + "ok": true, + "error": null, + "completion_tokens": 550, + "reasoning_chars": 832, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/108", + "ok": true, + "error": null, + "completion_tokens": 927, + "reasoning_chars": 2239, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/109", + "ok": true, + "error": null, + "completion_tokens": 896, + "reasoning_chars": 1798, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/110", + "ok": true, + "error": null, + "completion_tokens": 814, + "reasoning_chars": 2036, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/111", + "ok": true, + "error": null, + "completion_tokens": 682, + "reasoning_chars": 1908, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/112", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 601, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/113", + "ok": true, + "error": null, + "completion_tokens": 1087, + "reasoning_chars": 2838, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/114", + "ok": true, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/115", + "ok": true, + "error": null, + "completion_tokens": 526, + "reasoning_chars": 669, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/116", + "ok": true, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/117", + "ok": true, + "error": null, + "completion_tokens": 627, + "reasoning_chars": 1390, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/118", + "ok": true, + "error": null, + "completion_tokens": 746, + "reasoning_chars": 2012, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/119", + "ok": true, + "error": null, + "completion_tokens": 771, + "reasoning_chars": 1930, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/120", + "ok": true, + "error": null, + "completion_tokens": 478, + "reasoning_chars": 833, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/121", + "ok": true, + "error": null, + "completion_tokens": 321, + "reasoning_chars": 546, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/122", + "ok": true, + "error": null, + "completion_tokens": 474, + "reasoning_chars": 1127, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/123", + "ok": true, + "error": null, + "completion_tokens": 1156, + "reasoning_chars": 2823, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/124", + "ok": true, + "error": null, + "completion_tokens": 237, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/125", + "ok": true, + "error": null, + "completion_tokens": 70, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/126", + "ok": true, + "error": null, + "completion_tokens": 898, + "reasoning_chars": 2203, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/127", + "ok": true, + "error": null, + "completion_tokens": 688, + "reasoning_chars": 1931, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/128", + "ok": true, + "error": null, + "completion_tokens": 553, + "reasoning_chars": 1255, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/129", + "ok": true, + "error": null, + "completion_tokens": 2998, + "reasoning_chars": 10153, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/130", + "ok": false, + "error": null, + "completion_tokens": 5698, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/131", + "ok": true, + "error": null, + "completion_tokens": 414, + "reasoning_chars": 1079, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/132", + "ok": true, + "error": null, + "completion_tokens": 24603, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/133", + "ok": true, + "error": null, + "completion_tokens": 786, + "reasoning_chars": 2176, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/134", + "ok": true, + "error": null, + "completion_tokens": 2303, + "reasoning_chars": 7582, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/135", + "ok": true, + "error": null, + "completion_tokens": 441, + "reasoning_chars": 1132, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/136", + "ok": true, + "error": null, + "completion_tokens": 669, + "reasoning_chars": 1809, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/137", + "ok": true, + "error": null, + "completion_tokens": 747, + "reasoning_chars": 2197, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/138", + "ok": true, + "error": null, + "completion_tokens": 280, + "reasoning_chars": 653, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/139", + "ok": true, + "error": null, + "completion_tokens": 303, + "reasoning_chars": 643, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/140", + "ok": true, + "error": null, + "completion_tokens": 813, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/141", + "ok": true, + "error": null, + "completion_tokens": 105, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/142", + "ok": true, + "error": null, + "completion_tokens": 817, + "reasoning_chars": 1733, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/143", + "ok": true, + "error": null, + "completion_tokens": 452, + "reasoning_chars": 823, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/144", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 980, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/145", + "ok": false, + "error": null, + "completion_tokens": 22818, + "reasoning_chars": 80104, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/146", + "ok": true, + "error": null, + "completion_tokens": 1208, + "reasoning_chars": 3191, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/147", + "ok": true, + "error": null, + "completion_tokens": 2309, + "reasoning_chars": 5398, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/148", + "ok": true, + "error": null, + "completion_tokens": 719, + "reasoning_chars": 1658, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/149", + "ok": true, + "error": null, + "completion_tokens": 547, + "reasoning_chars": 1414, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/150", + "ok": true, + "error": null, + "completion_tokens": 384, + "reasoning_chars": 788, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/151", + "ok": true, + "error": null, + "completion_tokens": 763, + "reasoning_chars": 2123, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/152", + "ok": true, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/153", + "ok": true, + "error": null, + "completion_tokens": 552, + "reasoning_chars": 1882, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/154", + "ok": true, + "error": null, + "completion_tokens": 363, + "reasoning_chars": 775, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/155", + "ok": true, + "error": null, + "completion_tokens": 578, + "reasoning_chars": 1598, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/156", + "ok": true, + "error": null, + "completion_tokens": 543, + "reasoning_chars": 840, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/157", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/158", + "ok": true, + "error": null, + "completion_tokens": 568, + "reasoning_chars": 1687, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/159", + "ok": true, + "error": null, + "completion_tokens": 448, + "reasoning_chars": 500, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/160", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/161", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/162", + "ok": true, + "error": null, + "completion_tokens": 475, + "reasoning_chars": 1482, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/163", + "ok": true, + "error": null, + "completion_tokens": 3621, + "reasoning_chars": 10942, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "ifeval": { + "task": "ifeval", + "n": 100, + "hits": 63, + "accuracy_pct": 63.0, + "accuracy_ex_trunc_pct": 64.29, + "truncated": 2, + "completion_tokens_p50": 429, + "completion_tokens_p90": 2408, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 4262, + "reasoning_leaked": null, + "empty": 2, + "length_limited": 2, + "errors": 0, + "elapsed_s": 10560.6, + "gen_tokens": 162715, + "tok_per_s": 15.4, + "instruction_level_pct": 74.67, + "instructions_checked": 150, + "details": [ + { + "id": "ifeval-153", + "ok": false, + "error": null, + "completion_tokens": 570, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-258", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-368", + "ok": true, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-85", + "ok": true, + "error": null, + "completion_tokens": 561, + "reasoning_chars": 1845, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-246", + "ok": false, + "error": null, + "completion_tokens": 173, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-23", + "ok": true, + "error": null, + "completion_tokens": 3319, + "reasoning_chars": 10375, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-372", + "ok": true, + "error": null, + "completion_tokens": 1394, + "reasoning_chars": 3595, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-387", + "ok": true, + "error": null, + "completion_tokens": 972, + "reasoning_chars": 3450, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-138", + "ok": true, + "error": null, + "completion_tokens": 515, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-56", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-277", + "ok": true, + "error": null, + "completion_tokens": 1205, + "reasoning_chars": 3823, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-407", + "ok": false, + "error": null, + "completion_tokens": 564, + "reasoning_chars": 377, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-171", + "ok": true, + "error": null, + "completion_tokens": 67, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-96", + "ok": true, + "error": null, + "completion_tokens": 18399, + "reasoning_chars": 19495, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-134", + "ok": false, + "error": null, + "completion_tokens": 849, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-218", + "ok": true, + "error": null, + "completion_tokens": 1154, + "reasoning_chars": 3668, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-330", + "ok": false, + "error": null, + "completion_tokens": 2475, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-6", + "ok": true, + "error": null, + "completion_tokens": 502, + "reasoning_chars": 1307, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-143", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 429, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-121", + "ok": true, + "error": null, + "completion_tokens": 442, + "reasoning_chars": 1389, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-219", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-524", + "ok": true, + "error": null, + "completion_tokens": 751, + "reasoning_chars": 793, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-206", + "ok": true, + "error": null, + "completion_tokens": 247, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-144", + "ok": true, + "error": null, + "completion_tokens": 530, + "reasoning_chars": 1188, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-447", + "ok": false, + "error": null, + "completion_tokens": 276, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-492", + "ok": true, + "error": null, + "completion_tokens": 2231, + "reasoning_chars": 6727, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-485", + "ok": true, + "error": null, + "completion_tokens": 2408, + "reasoning_chars": 5909, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-220", + "ok": true, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-412", + "ok": false, + "error": null, + "completion_tokens": 1443, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-342", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 390, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-222", + "ok": true, + "error": null, + "completion_tokens": 450, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-131", + "ok": false, + "error": null, + "completion_tokens": 217, + "reasoning_chars": 633, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-210", + "ok": false, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-137", + "ok": true, + "error": null, + "completion_tokens": 429, + "reasoning_chars": 599, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-464", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-503", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-446", + "ok": true, + "error": null, + "completion_tokens": 523, + "reasoning_chars": 1246, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-2", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-525", + "ok": true, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 307, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-27", + "ok": true, + "error": null, + "completion_tokens": 88, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-136", + "ok": true, + "error": null, + "completion_tokens": 951, + "reasoning_chars": 1767, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-286", + "ok": false, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-449", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 1036, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-55", + "ok": true, + "error": null, + "completion_tokens": 646, + "reasoning_chars": 2051, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-93", + "ok": true, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-399", + "ok": true, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-19", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-229", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-181", + "ok": true, + "error": null, + "completion_tokens": 638, + "reasoning_chars": 2027, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-362", + "ok": true, + "error": null, + "completion_tokens": 120, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-289", + "ok": true, + "error": null, + "completion_tokens": 577, + "reasoning_chars": 1831, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-356", + "ok": true, + "error": null, + "completion_tokens": 714, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-115", + "ok": false, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-404", + "ok": false, + "error": null, + "completion_tokens": 724, + "reasoning_chars": 1272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-243", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 81920, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-90", + "ok": false, + "error": null, + "completion_tokens": 788, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-54", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-276", + "ok": true, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 246, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-308", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 610, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-64", + "ok": false, + "error": null, + "completion_tokens": 1215, + "reasoning_chars": 4262, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-260", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-337", + "ok": false, + "error": null, + "completion_tokens": 826, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-510", + "ok": true, + "error": null, + "completion_tokens": 854, + "reasoning_chars": 3383, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-119", + "ok": true, + "error": null, + "completion_tokens": 1817, + "reasoning_chars": 5508, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-5", + "ok": true, + "error": null, + "completion_tokens": 274, + "reasoning_chars": 825, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-468", + "ok": false, + "error": null, + "completion_tokens": 451, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-418", + "ok": false, + "error": null, + "completion_tokens": 591, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-476", + "ok": true, + "error": null, + "completion_tokens": 2313, + "reasoning_chars": 3914, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-49", + "ok": true, + "error": null, + "completion_tokens": 6619, + "reasoning_chars": 14592, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-336", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-422", + "ok": true, + "error": null, + "completion_tokens": 131, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-417", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 782, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-347", + "ok": false, + "error": null, + "completion_tokens": 235, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-101", + "ok": true, + "error": null, + "completion_tokens": 624, + "reasoning_chars": 1180, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-209", + "ok": false, + "error": null, + "completion_tokens": 430, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-341", + "ok": false, + "error": null, + "completion_tokens": 551, + "reasoning_chars": 1337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-533", + "ok": true, + "error": null, + "completion_tokens": 190, + "reasoning_chars": 557, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-158", + "ok": true, + "error": null, + "completion_tokens": 111, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-321", + "ok": true, + "error": null, + "completion_tokens": 1215, + "reasoning_chars": 5029, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-311", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-50", + "ok": false, + "error": null, + "completion_tokens": 9826, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-518", + "ok": true, + "error": null, + "completion_tokens": 3940, + "reasoning_chars": 6347, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-374", + "ok": true, + "error": null, + "completion_tokens": 348, + "reasoning_chars": 871, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-317", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 573, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-512", + "ok": true, + "error": null, + "completion_tokens": 977, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-102", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-72", + "ok": false, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-33", + "ok": false, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-263", + "ok": false, + "error": null, + "completion_tokens": 1120, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-501", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 76793, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-4", + "ok": true, + "error": null, + "completion_tokens": 343, + "reasoning_chars": 1184, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-444", + "ok": false, + "error": null, + "completion_tokens": 6348, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-228", + "ok": true, + "error": null, + "completion_tokens": 267, + "reasoning_chars": 186, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-299", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-278", + "ok": true, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-154", + "ok": false, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-301", + "ok": true, + "error": null, + "completion_tokens": 618, + "reasoning_chars": 1996, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-161", + "ok": false, + "error": null, + "completion_tokens": 3233, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-394", + "ok": false, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-159", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 478, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + } + ] + } + } + }, + "ablated-gguf": { + "started": "2026-10-10 18:55:48", + "finished": "2026-10-10 22:10:40", + "seed": 20260816, + "thinking": true, + "tasks": { + "mmlu": { + "task": "mmlu", + "n": 150, + "hits": 102, + "accuracy_pct": 68.0, + "accuracy_ex_trunc_pct": 68.0, + "truncated": 0, + "completion_tokens_p50": 46, + "completion_tokens_p90": 513, + "completion_tokens_max": 20591, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 1434, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 7938.9, + "gen_tokens": 62395, + "tok_per_s": 7.9, + "details": [ + { + "id": "mmlu-12325", + "ok": true, + "error": null, + "completion_tokens": 1129, + "reasoning_chars": 3046, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-154", + "ok": false, + "error": null, + "completion_tokens": 269, + "reasoning_chars": 842, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12844", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1153", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11089", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12782", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5422", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12363", + "ok": true, + "error": null, + "completion_tokens": 1810, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12556", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13163", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-270", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6943", + "ok": true, + "error": null, + "completion_tokens": 1201, + "reasoning_chars": 5199, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8330", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9276", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7584", + "ok": true, + "error": null, + "completion_tokens": 137, + "reasoning_chars": 367, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12819", + "ok": true, + "error": null, + "completion_tokens": 472, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6163", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 408, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4099", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 918, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10389", + "ok": false, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1539", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12013", + "ok": true, + "error": null, + "completion_tokens": 1239, + "reasoning_chars": 4554, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7814", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1106", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 160, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1616", + "ok": true, + "error": null, + "completion_tokens": 293, + "reasoning_chars": 578, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-555", + "ok": true, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 1013, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3698", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-618", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 566, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9842", + "ok": true, + "error": null, + "completion_tokens": 205, + "reasoning_chars": 590, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4229", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1022", + "ok": true, + "error": null, + "completion_tokens": 273, + "reasoning_chars": 1030, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6421", + "ok": false, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1997", + "ok": true, + "error": null, + "completion_tokens": 348, + "reasoning_chars": 1148, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8941", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-565", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11364", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5959", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13066", + "ok": false, + "error": null, + "completion_tokens": 476, + "reasoning_chars": 1685, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3122", + "ok": true, + "error": null, + "completion_tokens": 513, + "reasoning_chars": 910, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3722", + "ok": true, + "error": null, + "completion_tokens": 409, + "reasoning_chars": 1411, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13839", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12570", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9959", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 145, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10791", + "ok": false, + "error": null, + "completion_tokens": 847, + "reasoning_chars": 3997, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12268", + "ok": true, + "error": null, + "completion_tokens": 1656, + "reasoning_chars": 4391, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9051", + "ok": false, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12887", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2667", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3906", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10304", + "ok": true, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 115, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12927", + "ok": true, + "error": null, + "completion_tokens": 232, + "reasoning_chars": 703, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6337", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 98, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3907", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12888", + "ok": false, + "error": null, + "completion_tokens": 464, + "reasoning_chars": 1396, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10629", + "ok": true, + "error": null, + "completion_tokens": 426, + "reasoning_chars": 718, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5801", + "ok": true, + "error": null, + "completion_tokens": 278, + "reasoning_chars": 906, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2936", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7389", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9532", + "ok": false, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 372, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8705", + "ok": true, + "error": null, + "completion_tokens": 256, + "reasoning_chars": 820, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11776", + "ok": true, + "error": null, + "completion_tokens": 755, + "reasoning_chars": 2216, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6836", + "ok": false, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7856", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 512, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9697", + "ok": true, + "error": null, + "completion_tokens": 469, + "reasoning_chars": 1599, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10825", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11178", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11420", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4658", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 575, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5403", + "ok": false, + "error": null, + "completion_tokens": 397, + "reasoning_chars": 1281, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6160", + "ok": true, + "error": null, + "completion_tokens": 405, + "reasoning_chars": 1073, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6872", + "ok": false, + "error": null, + "completion_tokens": 10055, + "reasoning_chars": 41490, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9738", + "ok": true, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 173, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3076", + "ok": false, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13496", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11289", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-222", + "ok": false, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12180", + "ok": true, + "error": null, + "completion_tokens": 901, + "reasoning_chars": 2687, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1143", + "ok": true, + "error": null, + "completion_tokens": 747, + "reasoning_chars": 844, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13086", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-7038", + "ok": true, + "error": null, + "completion_tokens": 276, + "reasoning_chars": 834, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13587", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12349", + "ok": true, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 720, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11747", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9848", + "ok": true, + "error": null, + "completion_tokens": 848, + "reasoning_chars": 2894, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3438", + "ok": true, + "error": null, + "completion_tokens": 278, + "reasoning_chars": 1128, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-89", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4253", + "ok": true, + "error": null, + "completion_tokens": 54, + "reasoning_chars": 95, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5460", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2494", + "ok": false, + "error": null, + "completion_tokens": 20591, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11149", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-718", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5330", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10716", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5636", + "ok": false, + "error": null, + "completion_tokens": 817, + "reasoning_chars": 2431, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11832", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-95", + "ok": true, + "error": null, + "completion_tokens": 354, + "reasoning_chars": 1114, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13960", + "ok": true, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2072", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12030", + "ok": true, + "error": null, + "completion_tokens": 1474, + "reasoning_chars": 5516, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2931", + "ok": true, + "error": null, + "completion_tokens": 441, + "reasoning_chars": 957, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2441", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13666", + "ok": false, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6575", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11625", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13283", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4262", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12931", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12150", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3678", + "ok": true, + "error": null, + "completion_tokens": 324, + "reasoning_chars": 1151, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4593", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1563", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13396", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8049", + "ok": true, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8396", + "ok": true, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 982, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-8493", + "ok": true, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 1434, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13514", + "ok": true, + "error": null, + "completion_tokens": 474, + "reasoning_chars": 1420, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-3000", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 136, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1635", + "ok": true, + "error": null, + "completion_tokens": 317, + "reasoning_chars": 548, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1391", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-627", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1573", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 266, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6731", + "ok": true, + "error": null, + "completion_tokens": 347, + "reasoning_chars": 981, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11665", + "ok": true, + "error": null, + "completion_tokens": 1130, + "reasoning_chars": 3955, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2914", + "ok": false, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6875", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2184", + "ok": true, + "error": null, + "completion_tokens": 228, + "reasoning_chars": 380, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12801", + "ok": true, + "error": null, + "completion_tokens": 108, + "reasoning_chars": 273, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-6717", + "ok": true, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 488, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12464", + "ok": false, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2787", + "ok": true, + "error": null, + "completion_tokens": 428, + "reasoning_chars": 1045, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11549", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-842", + "ok": true, + "error": null, + "completion_tokens": 458, + "reasoning_chars": 2134, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13868", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9647", + "ok": true, + "error": null, + "completion_tokens": 355, + "reasoning_chars": 1375, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9502", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12441", + "ok": true, + "error": null, + "completion_tokens": 409, + "reasoning_chars": 1338, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-4392", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-5036", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 662, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12606", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-1400", + "ok": true, + "error": null, + "completion_tokens": 81, + "reasoning_chars": 371, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13979", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-13634", + "ok": true, + "error": null, + "completion_tokens": 205, + "reasoning_chars": 952, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-11226", + "ok": true, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-625", + "ok": false, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-257", + "ok": true, + "error": null, + "completion_tokens": 505, + "reasoning_chars": 1240, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-12853", + "ok": true, + "error": null, + "completion_tokens": 294, + "reasoning_chars": 563, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-451", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-2074", + "ok": true, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 41, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-9117", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 722, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlu-10748", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "cmmlu": { + "task": "cmmlu", + "n": 150, + "hits": 97, + "accuracy_pct": 64.67, + "accuracy_ex_trunc_pct": 64.67, + "truncated": 0, + "completion_tokens_p50": 44, + "completion_tokens_p90": 388, + "completion_tokens_max": 2764, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 697, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 8282.3, + "gen_tokens": 24182, + "tok_per_s": 2.9, + "details": [ + { + "id": "cmmlu-chinese_teacher_qualification-63", + "ok": true, + "error": null, + "completion_tokens": 185, + "reasoning_chars": 297, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-54", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 254, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-91", + "ok": false, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 420, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-260", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-32", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-29", + "ok": true, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 54, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-200", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-98", + "ok": true, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-44", + "ok": true, + "error": null, + "completion_tokens": 218, + "reasoning_chars": 344, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-118", + "ok": true, + "error": null, + "completion_tokens": 231, + "reasoning_chars": 762, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-116", + "ok": true, + "error": null, + "completion_tokens": 162, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-43", + "ok": true, + "error": null, + "completion_tokens": 388, + "reasoning_chars": 530, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-145", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-247", + "ok": true, + "error": null, + "completion_tokens": 176, + "reasoning_chars": 308, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-224", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 285, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-26", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-105", + "ok": true, + "error": null, + "completion_tokens": 1221, + "reasoning_chars": 1753, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-15", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-62", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-nutrition-74", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-181", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-127", + "ok": true, + "error": null, + "completion_tokens": 518, + "reasoning_chars": 911, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_geography-107", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-18", + "ok": true, + "error": null, + "completion_tokens": 332, + "reasoning_chars": 582, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-82", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 160, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_food_culture-38", + "ok": true, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 164, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-198", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 274, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-96", + "ok": true, + "error": null, + "completion_tokens": 172, + "reasoning_chars": 251, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-73", + "ok": true, + "error": null, + "completion_tokens": 613, + "reasoning_chars": 711, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-20", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-95", + "ok": true, + "error": null, + "completion_tokens": 123, + "reasoning_chars": 197, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-88", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-192", + "ok": true, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 697, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-66", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 233, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-100", + "ok": true, + "error": null, + "completion_tokens": 384, + "reasoning_chars": 406, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-136", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-16", + "ok": false, + "error": null, + "completion_tokens": 274, + "reasoning_chars": 1045, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-logical-90", + "ok": true, + "error": null, + "completion_tokens": 513, + "reasoning_chars": 1661, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-41", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-345", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-103", + "ok": false, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 217, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-93", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-88", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-206", + "ok": true, + "error": null, + "completion_tokens": 183, + "reasoning_chars": 303, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-105", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-90", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-190", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-international_law-39", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-155", + "ok": true, + "error": null, + "completion_tokens": 959, + "reasoning_chars": 3310, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-24", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-6", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-31", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-243", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-63", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-135", + "ok": false, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-89", + "ok": true, + "error": null, + "completion_tokens": 60, + "reasoning_chars": 82, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-91", + "ok": false, + "error": null, + "completion_tokens": 117, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-120", + "ok": true, + "error": null, + "completion_tokens": 619, + "reasoning_chars": 1065, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-62", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-106", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-food_science-106", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-24", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-157", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-41", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-115", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-journalism-29", + "ok": true, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 110, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-traditional_chinese_medicine-67", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 95, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-161", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 159, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-120", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-education-117", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 212, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-169", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 171, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-129", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-19", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-8", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-72", + "ok": true, + "error": null, + "completion_tokens": 298, + "reasoning_chars": 575, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-public_relations-139", + "ok": false, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 681, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_security-22", + "ok": true, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-7", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_law-108", + "ok": true, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 165, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-31", + "ok": true, + "error": null, + "completion_tokens": 28, + "reasoning_chars": 31, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-131", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-35", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_actuarial_science-38", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-165", + "ok": true, + "error": null, + "completion_tokens": 153, + "reasoning_chars": 215, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marketing-101", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-107", + "ok": true, + "error": null, + "completion_tokens": 84, + "reasoning_chars": 105, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-8", + "ok": false, + "error": null, + "completion_tokens": 2764, + "reasoning_chars": 11074, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medical_statistics-86", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-security_study-30", + "ok": true, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 59, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-clinical_knowledge-118", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_law-24", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-46", + "ok": true, + "error": null, + "completion_tokens": 618, + "reasoning_chars": 2018, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-anatomy-21", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-145", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-arts-91", + "ok": true, + "error": null, + "completion_tokens": 64, + "reasoning_chars": 84, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_chinese-109", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 9, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-machine_learning-44", + "ok": false, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-74", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_mathematics-63", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-astronomy-146", + "ok": true, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-109", + "ok": true, + "error": null, + "completion_tokens": 342, + "reasoning_chars": 633, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-construction_project_management-16", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-marxist_theory-113", + "ok": false, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 285, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-102", + "ok": false, + "error": null, + "completion_tokens": 38, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-207", + "ok": false, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-19", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-164", + "ok": true, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 399, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-338", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 202, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-7", + "ok": true, + "error": null, + "completion_tokens": 601, + "reasoning_chars": 1653, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-92", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-47", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 1171, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-astronomy-140", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-legal_and_moral_basis-100", + "ok": true, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 141, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-37", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_commonsense-191", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_medicine-34", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-88", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 278, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-public_relations-77", + "ok": false, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 110, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_history-306", + "ok": true, + "error": null, + "completion_tokens": 748, + "reasoning_chars": 2021, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-business_ethics-40", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 149, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_chemistry-7", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-152", + "ok": true, + "error": null, + "completion_tokens": 200, + "reasoning_chars": 339, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-1", + "ok": true, + "error": null, + "completion_tokens": 124, + "reasoning_chars": 178, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-94", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 411, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_psychology-109", + "ok": true, + "error": null, + "completion_tokens": 156, + "reasoning_chars": 257, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_literature-164", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 174, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sports_science-89", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-economics-17", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-arts-2", + "ok": true, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 212, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-51", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-agronomy-141", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 279, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-computer_science-196", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-genetics-97", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-jurisprudence-202", + "ok": true, + "error": null, + "completion_tokens": 564, + "reasoning_chars": 664, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-management-195", + "ok": true, + "error": null, + "completion_tokens": 336, + "reasoning_chars": 253, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-conceptual_physics-21", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-sociology-49", + "ok": true, + "error": null, + "completion_tokens": 63, + "reasoning_chars": 101, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_engineering_hydrology-94", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-ancient_chinese-8", + "ok": false, + "error": null, + "completion_tokens": 506, + "reasoning_chars": 629, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_medicine-98", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 169, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-modern_chinese-28", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_foreign_policy-74", + "ok": false, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_driving_rule-121", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 663, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_mathematics-120", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-elementary_information_and_technology-105", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 113, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-chinese_teacher_qualification-35", + "ok": true, + "error": null, + "completion_tokens": 181, + "reasoning_chars": 215, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-professional_accounting-51", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-college_mathematics-49", + "ok": true, + "error": null, + "completion_tokens": 1992, + "reasoning_chars": 4739, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-high_school_politics-130", + "ok": true, + "error": null, + "completion_tokens": 470, + "reasoning_chars": 1478, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "cmmlu-virology-140", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "mmlu_pro": { + "task": "mmlu_pro", + "n": 150, + "hits": 103, + "accuracy_pct": 68.67, + "accuracy_ex_trunc_pct": 69.13, + "truncated": 1, + "completion_tokens_p50": 67, + "completion_tokens_p90": 1654, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 3678, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 3046.9, + "gen_tokens": 137876, + "tok_per_s": 45.3, + "details": [ + { + "id": "mmlupro-5831", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2299", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9280", + "ok": true, + "error": null, + "completion_tokens": 152, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5486", + "ok": true, + "error": null, + "completion_tokens": 521, + "reasoning_chars": 2383, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5761", + "ok": true, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6589", + "ok": true, + "error": null, + "completion_tokens": 881, + "reasoning_chars": 2013, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5389", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5131", + "ok": false, + "error": null, + "completion_tokens": 38, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1677", + "ok": true, + "error": null, + "completion_tokens": 1254, + "reasoning_chars": 4343, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4388", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 46905, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "mmlupro-3285", + "ok": true, + "error": null, + "completion_tokens": 1010, + "reasoning_chars": 1506, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4118", + "ok": true, + "error": null, + "completion_tokens": 900, + "reasoning_chars": 2304, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5344", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11696", + "ok": false, + "error": null, + "completion_tokens": 1437, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-858", + "ok": true, + "error": null, + "completion_tokens": 100, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-214", + "ok": true, + "error": null, + "completion_tokens": 1044, + "reasoning_chars": 1793, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6609", + "ok": true, + "error": null, + "completion_tokens": 743, + "reasoning_chars": 1828, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2083", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9752", + "ok": false, + "error": null, + "completion_tokens": 6662, + "reasoning_chars": 17528, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7213", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5234", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8570", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 293, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5612", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2071", + "ok": true, + "error": null, + "completion_tokens": 683, + "reasoning_chars": 2335, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9842", + "ok": true, + "error": null, + "completion_tokens": 54, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4665", + "ok": true, + "error": null, + "completion_tokens": 916, + "reasoning_chars": 2200, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5564", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6475", + "ok": true, + "error": null, + "completion_tokens": 413, + "reasoning_chars": 900, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2153", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5812", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 141, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3407", + "ok": true, + "error": null, + "completion_tokens": 2969, + "reasoning_chars": 5560, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2879", + "ok": false, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 423, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8266", + "ok": true, + "error": null, + "completion_tokens": 389, + "reasoning_chars": 516, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1916", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3486", + "ok": false, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11275", + "ok": true, + "error": null, + "completion_tokens": 357, + "reasoning_chars": 999, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6040", + "ok": true, + "error": null, + "completion_tokens": 1668, + "reasoning_chars": 4756, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8565", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 261, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6848", + "ok": false, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8926", + "ok": true, + "error": null, + "completion_tokens": 249, + "reasoning_chars": 297, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8458", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7854", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3962", + "ok": true, + "error": null, + "completion_tokens": 3531, + "reasoning_chars": 4305, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6470", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3813", + "ok": true, + "error": null, + "completion_tokens": 371, + "reasoning_chars": 1087, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8026", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11030", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8151", + "ok": true, + "error": null, + "completion_tokens": 62, + "reasoning_chars": 81, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5286", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3888", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11676", + "ok": true, + "error": null, + "completion_tokens": 3148, + "reasoning_chars": 5730, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8762", + "ok": false, + "error": null, + "completion_tokens": 1654, + "reasoning_chars": 2863, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-12160", + "ok": true, + "error": null, + "completion_tokens": 563, + "reasoning_chars": 657, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4022", + "ok": true, + "error": null, + "completion_tokens": 2506, + "reasoning_chars": 3196, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10092", + "ok": true, + "error": null, + "completion_tokens": 598, + "reasoning_chars": 1372, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7364", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6001", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10873", + "ok": true, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9777", + "ok": true, + "error": null, + "completion_tokens": 10547, + "reasoning_chars": 23294, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8723", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4595", + "ok": true, + "error": null, + "completion_tokens": 402, + "reasoning_chars": 999, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8423", + "ok": true, + "error": null, + "completion_tokens": 3254, + "reasoning_chars": 4944, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11051", + "ok": false, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 557, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-836", + "ok": false, + "error": null, + "completion_tokens": 30, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11297", + "ok": true, + "error": null, + "completion_tokens": 13993, + "reasoning_chars": 36076, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3950", + "ok": true, + "error": null, + "completion_tokens": 274, + "reasoning_chars": 612, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1562", + "ok": false, + "error": null, + "completion_tokens": 933, + "reasoning_chars": 3678, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2041", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3473", + "ok": true, + "error": null, + "completion_tokens": 286, + "reasoning_chars": 1288, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11253", + "ok": true, + "error": null, + "completion_tokens": 713, + "reasoning_chars": 2502, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3858", + "ok": true, + "error": null, + "completion_tokens": 548, + "reasoning_chars": 1016, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2231", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2190", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6930", + "ok": true, + "error": null, + "completion_tokens": 457, + "reasoning_chars": 1712, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6558", + "ok": true, + "error": null, + "completion_tokens": 592, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5371", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1766", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6045", + "ok": true, + "error": null, + "completion_tokens": 1161, + "reasoning_chars": 3051, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2298", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6400", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6499", + "ok": true, + "error": null, + "completion_tokens": 553, + "reasoning_chars": 1393, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7936", + "ok": true, + "error": null, + "completion_tokens": 250, + "reasoning_chars": 1000, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8566", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11633", + "ok": false, + "error": null, + "completion_tokens": 68, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9843", + "ok": false, + "error": null, + "completion_tokens": 8115, + "reasoning_chars": 20744, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11758", + "ok": false, + "error": null, + "completion_tokens": 339, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2843", + "ok": true, + "error": null, + "completion_tokens": 1253, + "reasoning_chars": 3282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7483", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11379", + "ok": true, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4866", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-818", + "ok": false, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2785", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8119", + "ok": false, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2214", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5774", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6049", + "ok": false, + "error": null, + "completion_tokens": 422, + "reasoning_chars": 1096, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7219", + "ok": true, + "error": null, + "completion_tokens": 492, + "reasoning_chars": 1803, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1660", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4767", + "ok": true, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5964", + "ok": false, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2743", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4062", + "ok": true, + "error": null, + "completion_tokens": 237, + "reasoning_chars": 637, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7368", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-827", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11877", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7557", + "ok": false, + "error": null, + "completion_tokens": 943, + "reasoning_chars": 3302, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9998", + "ok": true, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 612, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10098", + "ok": true, + "error": null, + "completion_tokens": 761, + "reasoning_chars": 1857, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1744", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9210", + "ok": true, + "error": null, + "completion_tokens": 372, + "reasoning_chars": 464, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11248", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 874, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11187", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2799", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4479", + "ok": true, + "error": null, + "completion_tokens": 1894, + "reasoning_chars": 4345, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10223", + "ok": true, + "error": null, + "completion_tokens": 573, + "reasoning_chars": 1294, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5698", + "ok": false, + "error": null, + "completion_tokens": 29, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8291", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3516", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5466", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4851", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-733", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-6995", + "ok": true, + "error": null, + "completion_tokens": 48, + "reasoning_chars": 169, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2573", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8481", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-5938", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10708", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8476", + "ok": false, + "error": null, + "completion_tokens": 56, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-7990", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 742, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2409", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11551", + "ok": false, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-674", + "ok": true, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 211, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-1978", + "ok": true, + "error": null, + "completion_tokens": 1149, + "reasoning_chars": 4029, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10420", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3445", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9488", + "ok": true, + "error": null, + "completion_tokens": 337, + "reasoning_chars": 814, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4993", + "ok": true, + "error": null, + "completion_tokens": 454, + "reasoning_chars": 1470, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4316", + "ok": true, + "error": null, + "completion_tokens": 222, + "reasoning_chars": 310, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4730", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11570", + "ok": false, + "error": null, + "completion_tokens": 67, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-2149", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8322", + "ok": true, + "error": null, + "completion_tokens": 2791, + "reasoning_chars": 7219, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-10993", + "ok": false, + "error": null, + "completion_tokens": 822, + "reasoning_chars": 2421, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4588", + "ok": true, + "error": null, + "completion_tokens": 4721, + "reasoning_chars": 10208, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-3258", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-8629", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-637", + "ok": false, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-4257", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 518, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9719", + "ok": true, + "error": null, + "completion_tokens": 799, + "reasoning_chars": 1236, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-11714", + "ok": false, + "error": null, + "completion_tokens": 4819, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "mmlupro-9678", + "ok": false, + "error": null, + "completion_tokens": 1001, + "reasoning_chars": 3378, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "ceval": { + "task": "ceval", + "n": 150, + "hits": 105, + "accuracy_pct": 70.0, + "accuracy_ex_trunc_pct": 70.47, + "truncated": 1, + "completion_tokens_p50": 41, + "completion_tokens_p90": 632, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 1483, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 7987.8, + "gen_tokens": 80359, + "tok_per_s": 10.1, + "details": [ + { + "id": "ceval-high_school_chinese-1", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-10", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-41", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_physics-3", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-21", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 175, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-4", + "ok": true, + "error": null, + "completion_tokens": 1416, + "reasoning_chars": 3209, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-21", + "ok": true, + "error": null, + "completion_tokens": 156, + "reasoning_chars": 257, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-2", + "ok": true, + "error": null, + "completion_tokens": 80, + "reasoning_chars": 124, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-6", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-12", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-19", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_chemistry-9", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-15", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-22", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 22, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-16", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 54, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-11", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-18", + "ok": true, + "error": null, + "completion_tokens": 93, + "reasoning_chars": 158, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-24", + "ok": true, + "error": null, + "completion_tokens": 587, + "reasoning_chars": 1834, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-29", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-2", + "ok": true, + "error": null, + "completion_tokens": 1204, + "reasoning_chars": 1483, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-11", + "ok": false, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 114, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-18", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-0", + "ok": false, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 52, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-14", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-27", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-art_studies-19", + "ok": true, + "error": null, + "completion_tokens": 191, + "reasoning_chars": 245, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-21", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-33", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 231, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-20", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-10", + "ok": true, + "error": null, + "completion_tokens": 399, + "reasoning_chars": 706, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-19", + "ok": true, + "error": null, + "completion_tokens": 51, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_network-7", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 99, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-41", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-44", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 254, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-9", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-28", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-39", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-1", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-24", + "ok": true, + "error": null, + "completion_tokens": 156, + "reasoning_chars": 272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-5", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_architecture-15", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-39", + "ok": true, + "error": null, + "completion_tokens": 400, + "reasoning_chars": 1819, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-23", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-7", + "ok": false, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-17", + "ok": true, + "error": null, + "completion_tokens": 256, + "reasoning_chars": 416, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_chemistry-6", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-15", + "ok": true, + "error": null, + "completion_tokens": 71, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-4", + "ok": true, + "error": null, + "completion_tokens": 7936, + "reasoning_chars": 21572, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-5", + "ok": true, + "error": null, + "completion_tokens": 1039, + "reasoning_chars": 3279, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-50", + "ok": true, + "error": null, + "completion_tokens": 136, + "reasoning_chars": 169, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_politics-11", + "ok": false, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-32", + "ok": false, + "error": null, + "completion_tokens": 455, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-16", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-0", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-sports_science-9", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-art_studies-15", + "ok": true, + "error": null, + "completion_tokens": 55, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-2", + "ok": false, + "error": null, + "completion_tokens": 9547, + "reasoning_chars": 24634, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-7", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 263, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-11", + "ok": true, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 45, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-operating_system-18", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 103, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-4", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-17", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-1", + "ok": true, + "error": null, + "completion_tokens": 143, + "reasoning_chars": 268, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-32", + "ok": true, + "error": null, + "completion_tokens": 110, + "reasoning_chars": 210, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-33", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-25", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-40", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 197, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-7", + "ok": true, + "error": null, + "completion_tokens": 907, + "reasoning_chars": 1215, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-5", + "ok": true, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 1934, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-12", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-4", + "ok": true, + "error": null, + "completion_tokens": 216, + "reasoning_chars": 326, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-38", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 66990, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "ceval-electrical_engineer-22", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-clinical_medicine-0", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-15", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-45", + "ok": true, + "error": null, + "completion_tokens": 91, + "reasoning_chars": 103, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-6", + "ok": false, + "error": null, + "completion_tokens": 40, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-legal_professional-20", + "ok": false, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-basic_medicine-2", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-17", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-16", + "ok": false, + "error": null, + "completion_tokens": 443, + "reasoning_chars": 1774, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-7", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_biology-16", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-10", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 943, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-32", + "ok": true, + "error": null, + "completion_tokens": 5058, + "reasoning_chars": 13264, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_geography-0", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-12", + "ok": true, + "error": null, + "completion_tokens": 1004, + "reasoning_chars": 1324, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-logic-14", + "ok": true, + "error": null, + "completion_tokens": 711, + "reasoning_chars": 2424, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-26", + "ok": true, + "error": null, + "completion_tokens": 632, + "reasoning_chars": 578, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-3", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-3", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 18, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-computer_network-13", + "ok": true, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 41, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-sports_science-5", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-electrical_engineer-27", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-fire_engineer-21", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-10", + "ok": true, + "error": null, + "completion_tokens": 131, + "reasoning_chars": 167, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-16", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-49", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-13", + "ok": true, + "error": null, + "completion_tokens": 1090, + "reasoning_chars": 3934, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-business_administration-5", + "ok": false, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-2", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 58, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-27", + "ok": false, + "error": null, + "completion_tokens": 333, + "reasoning_chars": 646, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_biology-8", + "ok": true, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 129, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_physics-14", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-14", + "ok": true, + "error": null, + "completion_tokens": 132, + "reasoning_chars": 182, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-31", + "ok": true, + "error": null, + "completion_tokens": 568, + "reasoning_chars": 1041, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-basic_medicine-13", + "ok": true, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 73, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-13", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_programming-12", + "ok": true, + "error": null, + "completion_tokens": 480, + "reasoning_chars": 1257, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-civil_servant-44", + "ok": true, + "error": null, + "completion_tokens": 1350, + "reasoning_chars": 3916, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-3", + "ok": true, + "error": null, + "completion_tokens": 1774, + "reasoning_chars": 4638, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-13", + "ok": true, + "error": null, + "completion_tokens": 800, + "reasoning_chars": 814, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_mathematics-6", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_politics-19", + "ok": true, + "error": null, + "completion_tokens": 321, + "reasoning_chars": 1133, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-environmental_impact_assessment_engineer-27", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-10", + "ok": true, + "error": null, + "completion_tokens": 252, + "reasoning_chars": 315, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-law-0", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-3", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 254, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-modern_chinese_history-6", + "ok": true, + "error": null, + "completion_tokens": 300, + "reasoning_chars": 274, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_chinese-13", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-urban_and_rural_planner-43", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-42", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_mathematics-13", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-discrete_mathematics-1", + "ok": true, + "error": null, + "completion_tokens": 51, + "reasoning_chars": 65, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-11", + "ok": true, + "error": null, + "completion_tokens": 853, + "reasoning_chars": 1063, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 107, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-plant_protection-2", + "ok": true, + "error": null, + "completion_tokens": 156, + "reasoning_chars": 33, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-probability_and_statistics-1", + "ok": true, + "error": null, + "completion_tokens": 262, + "reasoning_chars": 618, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_physics-16", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_chemistry-11", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-teacher_qualification-5", + "ok": true, + "error": null, + "completion_tokens": 214, + "reasoning_chars": 392, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-40", + "ok": true, + "error": null, + "completion_tokens": 506, + "reasoning_chars": 1631, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_physics-6", + "ok": false, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-middle_school_mathematics-10", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-ideological_and_moral_cultivation-10", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-17", + "ok": false, + "error": null, + "completion_tokens": 394, + "reasoning_chars": 578, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-tax_accountant-11", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-professional_tour_guide-16", + "ok": true, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 64, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-education_science-1", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_physics-0", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-37", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-9", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 18, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-high_school_history-16", + "ok": true, + "error": null, + "completion_tokens": 260, + "reasoning_chars": 353, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-chinese_language_and_literature-5", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-advanced_mathematics-18", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-physician-30", + "ok": true, + "error": null, + "completion_tokens": 99, + "reasoning_chars": 168, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-accountant-21", + "ok": true, + "error": null, + "completion_tokens": 250, + "reasoning_chars": 319, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_economics-42", + "ok": false, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-veterinary_medicine-5", + "ok": false, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "ceval-college_chemistry-4", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "arc": { + "task": "arc", + "n": 150, + "hits": 137, + "accuracy_pct": 91.33, + "accuracy_ex_trunc_pct": 91.33, + "truncated": 0, + "completion_tokens_p50": 247, + "completion_tokens_p90": 471, + "completion_tokens_max": 2134, + "reasoning_chars_p50": 763, + "reasoning_chars_p90": 1429, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 8042.1, + "gen_tokens": 40490, + "tok_per_s": 5.0, + "details": [ + { + "id": "arc-Mercury_415084", + "ok": true, + "error": null, + "completion_tokens": 94, + "reasoning_chars": 58, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7249708", + "ok": true, + "error": null, + "completion_tokens": 437, + "reasoning_chars": 1290, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_4_24", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 1080, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7056823", + "ok": true, + "error": null, + "completion_tokens": 310, + "reasoning_chars": 899, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415534", + "ok": true, + "error": null, + "completion_tokens": 250, + "reasoning_chars": 682, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405827", + "ok": true, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7263183", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 684, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7106960", + "ok": true, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 186, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_7", + "ok": true, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 246, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7182245", + "ok": true, + "error": null, + "completion_tokens": 523, + "reasoning_chars": 1692, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416177", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-ACTAAP_2011_5_8", + "ok": true, + "error": null, + "completion_tokens": 346, + "reasoning_chars": 1066, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_8_36", + "ok": true, + "error": null, + "completion_tokens": 366, + "reasoning_chars": 1059, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401699", + "ok": true, + "error": null, + "completion_tokens": 103, + "reasoning_chars": 298, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7107240", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 290, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_4_26", + "ok": true, + "error": null, + "completion_tokens": 321, + "reasoning_chars": 728, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402638", + "ok": true, + "error": null, + "completion_tokens": 564, + "reasoning_chars": 1812, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7026723", + "ok": true, + "error": null, + "completion_tokens": 40, + "reasoning_chars": 163, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415422", + "ok": true, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 877, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2000_8_2", + "ok": true, + "error": null, + "completion_tokens": 340, + "reasoning_chars": 1065, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2006_9_4", + "ok": true, + "error": null, + "completion_tokens": 626, + "reasoning_chars": 1695, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401272", + "ok": true, + "error": null, + "completion_tokens": 93, + "reasoning_chars": 408, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416169", + "ok": true, + "error": null, + "completion_tokens": 247, + "reasoning_chars": 743, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7116235", + "ok": true, + "error": null, + "completion_tokens": 345, + "reasoning_chars": 1377, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7109463", + "ok": true, + "error": null, + "completion_tokens": 246, + "reasoning_chars": 609, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7032428", + "ok": false, + "error": null, + "completion_tokens": 738, + "reasoning_chars": 2238, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AIMS_2008_8_11", + "ok": true, + "error": null, + "completion_tokens": 325, + "reasoning_chars": 909, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_410334", + "ok": true, + "error": null, + "completion_tokens": 326, + "reasoning_chars": 1112, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_36", + "ok": true, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 389, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_415719", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 496, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_17", + "ok": true, + "error": null, + "completion_tokens": 154, + "reasoning_chars": 763, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402256", + "ok": true, + "error": null, + "completion_tokens": 92, + "reasoning_chars": 345, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_400443", + "ok": false, + "error": null, + "completion_tokens": 20, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7133245", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 693, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413458", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401372", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 728, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401116", + "ok": true, + "error": null, + "completion_tokens": 222, + "reasoning_chars": 664, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP__5_10312", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 904, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2007_4_10280", + "ok": true, + "error": null, + "completion_tokens": 186, + "reasoning_chars": 439, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_416683", + "ok": true, + "error": null, + "completion_tokens": 453, + "reasoning_chars": 1223, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7190120", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 910, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401295", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 1352, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_4_28", + "ok": true, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 347, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7223493", + "ok": true, + "error": null, + "completion_tokens": 319, + "reasoning_chars": 1081, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7018410", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 365, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1998_8_24", + "ok": true, + "error": null, + "completion_tokens": 566, + "reasoning_chars": 1703, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2003_8_31", + "ok": true, + "error": null, + "completion_tokens": 320, + "reasoning_chars": 903, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_405501", + "ok": true, + "error": null, + "completion_tokens": 384, + "reasoning_chars": 1087, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408030", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 1003, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401684", + "ok": true, + "error": null, + "completion_tokens": 284, + "reasoning_chars": 1256, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7041668", + "ok": true, + "error": null, + "completion_tokens": 356, + "reasoning_chars": 977, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7205870", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 915, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7210140", + "ok": true, + "error": null, + "completion_tokens": 474, + "reasoning_chars": 1482, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_405873", + "ok": true, + "error": null, + "completion_tokens": 388, + "reasoning_chars": 1312, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400052", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7018270", + "ok": true, + "error": null, + "completion_tokens": 562, + "reasoning_chars": 1732, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_406550", + "ok": true, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 772, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7150728", + "ok": true, + "error": null, + "completion_tokens": 471, + "reasoning_chars": 1311, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400839", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 681, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_30", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 807, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400383", + "ok": false, + "error": null, + "completion_tokens": 480, + "reasoning_chars": 1368, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7236740", + "ok": true, + "error": null, + "completion_tokens": 208, + "reasoning_chars": 300, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-CSZ_2005_5_CSZ10021", + "ok": true, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 775, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7093958", + "ok": true, + "error": null, + "completion_tokens": 266, + "reasoning_chars": 792, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7026355", + "ok": true, + "error": null, + "completion_tokens": 48, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408324", + "ok": true, + "error": null, + "completion_tokens": 148, + "reasoning_chars": 338, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP__7_10348", + "ok": true, + "error": null, + "completion_tokens": 155, + "reasoning_chars": 315, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402629", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 469, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400046", + "ok": true, + "error": null, + "completion_tokens": 421, + "reasoning_chars": 1141, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7180810", + "ok": true, + "error": null, + "completion_tokens": 399, + "reasoning_chars": 1081, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_180005", + "ok": true, + "error": null, + "completion_tokens": 162, + "reasoning_chars": 675, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AIMS_2009_8_14", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2007_5_16", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_LBS10662", + "ok": true, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 352, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_23", + "ok": true, + "error": null, + "completion_tokens": 233, + "reasoning_chars": 756, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2012_8_19", + "ok": false, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 524, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7041650", + "ok": true, + "error": null, + "completion_tokens": 363, + "reasoning_chars": 852, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416181", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 389, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TIMSS_1995_8_J6", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 1099, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2014_5_13", + "ok": true, + "error": null, + "completion_tokens": 365, + "reasoning_chars": 1432, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7106750", + "ok": true, + "error": null, + "completion_tokens": 381, + "reasoning_chars": 1041, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1999_4_23", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 603, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MEA_2013_5_12", + "ok": true, + "error": null, + "completion_tokens": 118, + "reasoning_chars": 195, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_8_42", + "ok": false, + "error": null, + "completion_tokens": 163, + "reasoning_chars": 419, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_408367", + "ok": true, + "error": null, + "completion_tokens": 142, + "reasoning_chars": 376, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7250058", + "ok": true, + "error": null, + "completion_tokens": 121, + "reasoning_chars": 559, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_414041", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 304, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401290", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 1077, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_409295", + "ok": true, + "error": null, + "completion_tokens": 326, + "reasoning_chars": 1112, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7211680", + "ok": true, + "error": null, + "completion_tokens": 463, + "reasoning_chars": 1659, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2009_5_37", + "ok": true, + "error": null, + "completion_tokens": 358, + "reasoning_chars": 1051, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413439", + "ok": true, + "error": null, + "completion_tokens": 436, + "reasoning_chars": 1275, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2013_8_32", + "ok": true, + "error": null, + "completion_tokens": 283, + "reasoning_chars": 677, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7216423", + "ok": true, + "error": null, + "completion_tokens": 326, + "reasoning_chars": 1022, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-ACTAAP_2015_7_9", + "ok": true, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 179, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7230405", + "ok": true, + "error": null, + "completion_tokens": 561, + "reasoning_chars": 1694, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7025008", + "ok": true, + "error": null, + "completion_tokens": 508, + "reasoning_chars": 1445, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_416536", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 527, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TIMSS_2007_8_pg60", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2009_4_34", + "ok": true, + "error": null, + "completion_tokens": 475, + "reasoning_chars": 1429, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_406833", + "ok": true, + "error": null, + "completion_tokens": 363, + "reasoning_chars": 1034, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_409647", + "ok": true, + "error": null, + "completion_tokens": 417, + "reasoning_chars": 1146, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_184363", + "ok": true, + "error": null, + "completion_tokens": 218, + "reasoning_chars": 548, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7107170", + "ok": true, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 736, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_416104", + "ok": true, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 248, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_10-v1", + "ok": true, + "error": null, + "completion_tokens": 75, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7202843", + "ok": true, + "error": null, + "completion_tokens": 539, + "reasoning_chars": 1445, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7029803", + "ok": true, + "error": null, + "completion_tokens": 373, + "reasoning_chars": 1113, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7164623", + "ok": true, + "error": null, + "completion_tokens": 268, + "reasoning_chars": 889, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-TAKS_2009_5_26", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7263060", + "ok": true, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 1065, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7268328", + "ok": true, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 778, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_402282", + "ok": true, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 290, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7029558", + "ok": true, + "error": null, + "completion_tokens": 50, + "reasoning_chars": 205, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2013_4_21", + "ok": true, + "error": null, + "completion_tokens": 146, + "reasoning_chars": 397, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7186358", + "ok": true, + "error": null, + "completion_tokens": 433, + "reasoning_chars": 1327, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-LEAP_2011_8_10434", + "ok": true, + "error": null, + "completion_tokens": 424, + "reasoning_chars": 1159, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2012_8_23651", + "ok": true, + "error": null, + "completion_tokens": 240, + "reasoning_chars": 624, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_407125", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 577, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7268258", + "ok": false, + "error": null, + "completion_tokens": 65, + "reasoning_chars": 284, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7027405", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 775, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_8_7", + "ok": true, + "error": null, + "completion_tokens": 315, + "reasoning_chars": 1498, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_191625", + "ok": true, + "error": null, + "completion_tokens": 2134, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2005_9_6", + "ok": true, + "error": null, + "completion_tokens": 161, + "reasoning_chars": 186, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7219678", + "ok": true, + "error": null, + "completion_tokens": 259, + "reasoning_chars": 1049, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_401678", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 249, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_401312", + "ok": true, + "error": null, + "completion_tokens": 202, + "reasoning_chars": 680, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2010_8_42", + "ok": false, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 165, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7030468", + "ok": true, + "error": null, + "completion_tokens": 404, + "reasoning_chars": 1151, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_175385", + "ok": true, + "error": null, + "completion_tokens": 396, + "reasoning_chars": 1132, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_175560", + "ok": true, + "error": null, + "completion_tokens": 97, + "reasoning_chars": 210, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_2004_9_5", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 137, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_409009", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 825, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7128923", + "ok": false, + "error": null, + "completion_tokens": 689, + "reasoning_chars": 2348, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-NYSEDREGENTS_2008_8_9", + "ok": false, + "error": null, + "completion_tokens": 32, + "reasoning_chars": 113, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7042945", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 1070, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-AKDE&ED_2008_8_47", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 993, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7185343", + "ok": true, + "error": null, + "completion_tokens": 493, + "reasoning_chars": 1645, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7218400", + "ok": true, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 942, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7221988", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_405950", + "ok": true, + "error": null, + "completion_tokens": 74, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7222530", + "ok": true, + "error": null, + "completion_tokens": 385, + "reasoning_chars": 1063, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7102323", + "ok": true, + "error": null, + "completion_tokens": 177, + "reasoning_chars": 358, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MDSA_2009_5_39", + "ok": true, + "error": null, + "completion_tokens": 246, + "reasoning_chars": 617, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-MCAS_1999_8_5", + "ok": true, + "error": null, + "completion_tokens": 188, + "reasoning_chars": 770, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_400661", + "ok": true, + "error": null, + "completion_tokens": 413, + "reasoning_chars": 1104, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_7081603", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 663, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-VASoL_2008_3_25", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 599, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_SC_413628", + "ok": true, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 896, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "arc-Mercury_406427", + "ok": true, + "error": null, + "completion_tokens": 307, + "reasoning_chars": 1134, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "truthfulqa": { + "task": "truthfulqa", + "n": 150, + "hits": 91, + "accuracy_pct": 60.67, + "accuracy_ex_trunc_pct": 60.67, + "truncated": 0, + "completion_tokens_p50": 7, + "completion_tokens_p90": 210, + "completion_tokens_max": 1012, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 782, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 8001.9, + "gen_tokens": 8743, + "tok_per_s": 1.1, + "details": [ + { + "id": "tqa-210", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-76", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-461", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-218", + "ok": true, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-254", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-635", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-193", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-148", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-400", + "ok": false, + "error": null, + "completion_tokens": 281, + "reasoning_chars": 999, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-766", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-240", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-50", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-83", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-381", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-325", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-655", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-721", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-391", + "ok": true, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-401", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-625", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-80", + "ok": false, + "error": null, + "completion_tokens": 325, + "reasoning_chars": 786, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-439", + "ok": true, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-66", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-155", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-516", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-522", + "ok": true, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-47", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-343", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-126", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-77", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-613", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-590", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-674", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-438", + "ok": false, + "error": null, + "completion_tokens": 106, + "reasoning_chars": 415, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-503", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-639", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-111", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-289", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-705", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-373", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-363", + "ok": true, + "error": null, + "completion_tokens": 1012, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-792", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-113", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-74", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-634", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 809, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-804", + "ok": true, + "error": null, + "completion_tokens": 464, + "reasoning_chars": 1729, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-626", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-676", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-174", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-455", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-725", + "ok": false, + "error": null, + "completion_tokens": 454, + "reasoning_chars": 1538, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-296", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-535", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-172", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-23", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-43", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-38", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-97", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-657", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-814", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-346", + "ok": true, + "error": null, + "completion_tokens": 295, + "reasoning_chars": 1062, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-365", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-384", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-677", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-486", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-55", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-680", + "ok": true, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-510", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-128", + "ok": true, + "error": null, + "completion_tokens": 174, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-805", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-394", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-435", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-285", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-687", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 864, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-588", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-100", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-399", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-266", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-693", + "ok": true, + "error": null, + "completion_tokens": 167, + "reasoning_chars": 476, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-532", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-558", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-319", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-459", + "ok": true, + "error": null, + "completion_tokens": 340, + "reasoning_chars": 1109, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-675", + "ok": false, + "error": null, + "completion_tokens": 440, + "reasoning_chars": 1887, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-129", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-331", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-12", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-164", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-306", + "ok": true, + "error": null, + "completion_tokens": 65, + "reasoning_chars": 272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-275", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-418", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-264", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-94", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-389", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-427", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-269", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-643", + "ok": false, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 1062, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-7", + "ok": true, + "error": null, + "completion_tokens": 808, + "reasoning_chars": 2672, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-419", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-616", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-699", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-473", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-523", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-214", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-376", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-123", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-200", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-493", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-474", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-387", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-746", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-790", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-181", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-413", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-349", + "ok": false, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-434", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-601", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-489", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-286", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-488", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-759", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-273", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-59", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-315", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-425", + "ok": true, + "error": null, + "completion_tokens": 405, + "reasoning_chars": 782, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-58", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-691", + "ok": true, + "error": null, + "completion_tokens": 345, + "reasoning_chars": 1215, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-497", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-515", + "ok": false, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-398", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-670", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-170", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-334", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-307", + "ok": false, + "error": null, + "completion_tokens": 648, + "reasoning_chars": 1961, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-495", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 853, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-597", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-775", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-186", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-733", + "ok": false, + "error": null, + "completion_tokens": 67, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-308", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-5", + "ok": false, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-231", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 823, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-356", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-810", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-270", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-13", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-114", + "ok": true, + "error": null, + "completion_tokens": 78, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-4", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-417", + "ok": true, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "tqa-750", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "gsm8k": { + "task": "gsm8k", + "n": 100, + "hits": 47, + "accuracy_pct": 47.0, + "accuracy_ex_trunc_pct": 47.0, + "truncated": 0, + "completion_tokens_p50": 8, + "completion_tokens_p90": 198, + "completion_tokens_max": 2326, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 408, + "reasoning_leaked": null, + "empty": 0, + "length_limited": 0, + "errors": 0, + "elapsed_s": 8003.5, + "gen_tokens": 7134, + "tok_per_s": 0.9, + "details": [ + { + "id": "gsm8k-144", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-627", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 698, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-195", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-201", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-730", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-166", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1141", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-832", + "ok": true, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-53", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1042", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-609", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-318", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-892", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-204", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1120", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1040", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 436, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-556", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1015", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-206", + "ok": true, + "error": null, + "completion_tokens": 272, + "reasoning_chars": 623, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1058", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1044", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-131", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1154", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-158", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1149", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1156", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-60", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-918", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1010", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1032", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1053", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-577", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-935", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-692", + "ok": true, + "error": null, + "completion_tokens": 226, + "reasoning_chars": 442, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-153", + "ok": true, + "error": null, + "completion_tokens": 516, + "reasoning_chars": 1504, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-40", + "ok": true, + "error": null, + "completion_tokens": 254, + "reasoning_chars": 705, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-613", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-875", + "ok": true, + "error": null, + "completion_tokens": 49, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-189", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-800", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-169", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1169", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-211", + "ok": true, + "error": null, + "completion_tokens": 166, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1183", + "ok": true, + "error": null, + "completion_tokens": 2326, + "reasoning_chars": 9080, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-193", + "ok": true, + "error": null, + "completion_tokens": 210, + "reasoning_chars": 777, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-230", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-787", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-83", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1318", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-733", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1205", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-88", + "ok": false, + "error": null, + "completion_tokens": 67, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-509", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-311", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-874", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-287", + "ok": true, + "error": null, + "completion_tokens": 139, + "reasoning_chars": 351, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1055", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1246", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-975", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-280", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-541", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-974", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-827", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-125", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-453", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-942", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1199", + "ok": false, + "error": null, + "completion_tokens": 36, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-884", + "ok": true, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-855", + "ok": true, + "error": null, + "completion_tokens": 135, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1270", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-349", + "ok": false, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1231", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1106", + "ok": true, + "error": null, + "completion_tokens": 33, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-803", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-922", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-152", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-920", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-207", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-297", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-786", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-101", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1301", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-27", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1005", + "ok": true, + "error": null, + "completion_tokens": 462, + "reasoning_chars": 1220, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-659", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-340", + "ok": true, + "error": null, + "completion_tokens": 592, + "reasoning_chars": 2097, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-288", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-763", + "ok": true, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-840", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-742", + "ok": true, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-222", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-468", + "ok": true, + "error": null, + "completion_tokens": 45, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-220", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-662", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-269", + "ok": true, + "error": null, + "completion_tokens": 87, + "reasoning_chars": 325, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-1193", + "ok": false, + "error": null, + "completion_tokens": 12, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-270", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-114", + "ok": true, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-728", + "ok": false, + "error": null, + "completion_tokens": 6, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "gsm8k-396", + "ok": true, + "error": null, + "completion_tokens": 198, + "reasoning_chars": 408, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "math500": { + "task": "math500", + "n": 100, + "hits": 67, + "accuracy_pct": 67.0, + "accuracy_ex_trunc_pct": 67.68, + "truncated": 1, + "completion_tokens_p50": 356, + "completion_tokens_p90": 3016, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 575, + "reasoning_chars_p90": 5961, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 10767.6, + "gen_tokens": 135071, + "tok_per_s": 12.5, + "details": [ + { + "id": "math-test/algebra/1553.json", + "ok": true, + "error": null, + "completion_tokens": 475, + "reasoning_chars": 664, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/538.json", + "ok": true, + "error": null, + "completion_tokens": 521, + "reasoning_chars": 886, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/691.json", + "ok": true, + "error": null, + "completion_tokens": 15, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1090.json", + "ok": true, + "error": null, + "completion_tokens": 758, + "reasoning_chars": 1603, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/116.json", + "ok": true, + "error": null, + "completion_tokens": 201, + "reasoning_chars": 282, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1255.json", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 189, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1143.json", + "ok": true, + "error": null, + "completion_tokens": 728, + "reasoning_chars": 908, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1201.json", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 40262, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "math-test/intermediate_algebra/964.json", + "ok": true, + "error": null, + "completion_tokens": 3267, + "reasoning_chars": 5961, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/662.json", + "ok": false, + "error": null, + "completion_tokens": 2763, + "reasoning_chars": 4403, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/153.json", + "ok": false, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 350, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/276.json", + "ok": true, + "error": null, + "completion_tokens": 390, + "reasoning_chars": 545, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1840.json", + "ok": false, + "error": null, + "completion_tokens": 11, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1072.json", + "ok": false, + "error": null, + "completion_tokens": 437, + "reasoning_chars": 609, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1907.json", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/567.json", + "ok": true, + "error": null, + "completion_tokens": 377, + "reasoning_chars": 552, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1922.json", + "ok": true, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1779.json", + "ok": true, + "error": null, + "completion_tokens": 4947, + "reasoning_chars": 11548, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/951.json", + "ok": true, + "error": null, + "completion_tokens": 239, + "reasoning_chars": 318, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1197.json", + "ok": false, + "error": null, + "completion_tokens": 5097, + "reasoning_chars": 10481, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/853.json", + "ok": true, + "error": null, + "completion_tokens": 882, + "reasoning_chars": 1542, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/516.json", + "ok": false, + "error": null, + "completion_tokens": 521, + "reasoning_chars": 911, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1114.json", + "ok": false, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 575, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1065.json", + "ok": true, + "error": null, + "completion_tokens": 445, + "reasoning_chars": 524, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1232.json", + "ok": false, + "error": null, + "completion_tokens": 291, + "reasoning_chars": 305, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/769.json", + "ok": true, + "error": null, + "completion_tokens": 805, + "reasoning_chars": 856, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1791.json", + "ok": false, + "error": null, + "completion_tokens": 1869, + "reasoning_chars": 3832, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/246.json", + "ok": true, + "error": null, + "completion_tokens": 308, + "reasoning_chars": 484, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1166.json", + "ok": true, + "error": null, + "completion_tokens": 4106, + "reasoning_chars": 7824, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1410.json", + "ok": false, + "error": null, + "completion_tokens": 420, + "reasoning_chars": 705, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1686.json", + "ok": true, + "error": null, + "completion_tokens": 217, + "reasoning_chars": 422, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/754.json", + "ok": false, + "error": null, + "completion_tokens": 963, + "reasoning_chars": 1903, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/477.json", + "ok": true, + "error": null, + "completion_tokens": 386, + "reasoning_chars": 1334, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2680.json", + "ok": true, + "error": null, + "completion_tokens": 151, + "reasoning_chars": 464, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/23957.json", + "ok": true, + "error": null, + "completion_tokens": 149, + "reasoning_chars": 266, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1865.json", + "ok": true, + "error": null, + "completion_tokens": 679, + "reasoning_chars": 1618, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1014.json", + "ok": true, + "error": null, + "completion_tokens": 356, + "reasoning_chars": 745, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/431.json", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 246, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1425.json", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 232, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2584.json", + "ok": false, + "error": null, + "completion_tokens": 228, + "reasoning_chars": 309, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2470.json", + "ok": true, + "error": null, + "completion_tokens": 147, + "reasoning_chars": 50, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/547.json", + "ok": true, + "error": null, + "completion_tokens": 3016, + "reasoning_chars": 8666, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1105.json", + "ok": true, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 23, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/347.json", + "ok": true, + "error": null, + "completion_tokens": 243, + "reasoning_chars": 634, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1257.json", + "ok": true, + "error": null, + "completion_tokens": 845, + "reasoning_chars": 1576, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1126.json", + "ok": true, + "error": null, + "completion_tokens": 573, + "reasoning_chars": 993, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/239.json", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1961.json", + "ok": false, + "error": null, + "completion_tokens": 270, + "reasoning_chars": 631, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1388.json", + "ok": true, + "error": null, + "completion_tokens": 197, + "reasoning_chars": 231, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1078.json", + "ok": true, + "error": null, + "completion_tokens": 577, + "reasoning_chars": 1097, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2157.json", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 414, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/339.json", + "ok": true, + "error": null, + "completion_tokens": 361, + "reasoning_chars": 1295, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1761.json", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1960.json", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 238, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/2086.json", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1462.json", + "ok": true, + "error": null, + "completion_tokens": 2757, + "reasoning_chars": 4691, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/1060.json", + "ok": true, + "error": null, + "completion_tokens": 127, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/1055.json", + "ok": true, + "error": null, + "completion_tokens": 2301, + "reasoning_chars": 6425, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2517.json", + "ok": false, + "error": null, + "completion_tokens": 519, + "reasoning_chars": 1019, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/996.json", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/893.json", + "ok": true, + "error": null, + "completion_tokens": 431, + "reasoning_chars": 507, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/51.json", + "ok": false, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1291.json", + "ok": true, + "error": null, + "completion_tokens": 387, + "reasoning_chars": 808, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/694.json", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/711.json", + "ok": false, + "error": null, + "completion_tokens": 8904, + "reasoning_chars": 29317, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1655.json", + "ok": false, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 391, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1936.json", + "ok": true, + "error": null, + "completion_tokens": 41, + "reasoning_chars": 49, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1572.json", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 747, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/509.json", + "ok": false, + "error": null, + "completion_tokens": 672, + "reasoning_chars": 968, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/864.json", + "ok": true, + "error": null, + "completion_tokens": 73, + "reasoning_chars": 182, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1405.json", + "ok": true, + "error": null, + "completion_tokens": 1368, + "reasoning_chars": 2486, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/192.json", + "ok": true, + "error": null, + "completion_tokens": 128, + "reasoning_chars": 190, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2102.json", + "ok": true, + "error": null, + "completion_tokens": 345, + "reasoning_chars": 500, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/843.json", + "ok": true, + "error": null, + "completion_tokens": 415, + "reasoning_chars": 686, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/187.json", + "ok": true, + "error": null, + "completion_tokens": 182, + "reasoning_chars": 432, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/612.json", + "ok": true, + "error": null, + "completion_tokens": 443, + "reasoning_chars": 1629, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/615.json", + "ok": true, + "error": null, + "completion_tokens": 1944, + "reasoning_chars": 5655, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1908.json", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/928.json", + "ok": true, + "error": null, + "completion_tokens": 971, + "reasoning_chars": 2150, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/505.json", + "ok": true, + "error": null, + "completion_tokens": 44, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1930.json", + "ok": false, + "error": null, + "completion_tokens": 2893, + "reasoning_chars": 5553, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1934.json", + "ok": true, + "error": null, + "completion_tokens": 138, + "reasoning_chars": 201, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1139.json", + "ok": true, + "error": null, + "completion_tokens": 13208, + "reasoning_chars": 37605, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1000.json", + "ok": true, + "error": null, + "completion_tokens": 3736, + "reasoning_chars": 7900, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/183.json", + "ok": true, + "error": null, + "completion_tokens": 750, + "reasoning_chars": 1277, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/1275.json", + "ok": true, + "error": null, + "completion_tokens": 566, + "reasoning_chars": 1166, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2700.json", + "ok": false, + "error": null, + "completion_tokens": 511, + "reasoning_chars": 1306, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2257.json", + "ok": false, + "error": null, + "completion_tokens": 179, + "reasoning_chars": 399, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/14.json", + "ok": false, + "error": null, + "completion_tokens": 245, + "reasoning_chars": 686, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/1300.json", + "ok": false, + "error": null, + "completion_tokens": 702, + "reasoning_chars": 828, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/number_theory/515.json", + "ok": true, + "error": null, + "completion_tokens": 290, + "reasoning_chars": 473, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1423.json", + "ok": false, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/intermediate_algebra/1102.json", + "ok": true, + "error": null, + "completion_tokens": 888, + "reasoning_chars": 1896, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1646.json", + "ok": true, + "error": null, + "completion_tokens": 10531, + "reasoning_chars": 30800, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/1272.json", + "ok": false, + "error": null, + "completion_tokens": 130, + "reasoning_chars": 171, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/counting_and_probability/25149.json", + "ok": true, + "error": null, + "completion_tokens": 286, + "reasoning_chars": 440, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/precalculus/697.json", + "ok": false, + "error": null, + "completion_tokens": 3048, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/algebra/2036.json", + "ok": true, + "error": null, + "completion_tokens": 122, + "reasoning_chars": 277, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/prealgebra/2037.json", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "math-test/geometry/353.json", + "ok": true, + "error": null, + "completion_tokens": 290, + "reasoning_chars": 564, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "bbh": { + "task": "bbh", + "n": 150, + "hits": 82, + "accuracy_pct": 54.67, + "accuracy_ex_trunc_pct": 54.36, + "truncated": 1, + "completion_tokens_p50": 53, + "completion_tokens_p90": 663, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 2348, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 8270.4, + "gen_tokens": 96814, + "tok_per_s": 11.7, + "details": [ + { + "id": "bbh-sports_understanding-101", + "ok": false, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-60", + "ok": true, + "error": null, + "completion_tokens": 527, + "reasoning_chars": 1855, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-113", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-31", + "ok": true, + "error": null, + "completion_tokens": 1570, + "reasoning_chars": 5969, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-144", + "ok": true, + "error": null, + "completion_tokens": 663, + "reasoning_chars": 1853, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-28", + "ok": true, + "error": null, + "completion_tokens": 370, + "reasoning_chars": 464, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-116", + "ok": true, + "error": null, + "completion_tokens": 82, + "reasoning_chars": 148, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-67", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-3", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 765, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-29", + "ok": false, + "error": null, + "completion_tokens": 56, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-192", + "ok": true, + "error": null, + "completion_tokens": 348, + "reasoning_chars": 1206, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-196", + "ok": true, + "error": null, + "completion_tokens": 545, + "reasoning_chars": 2162, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-10", + "ok": false, + "error": null, + "completion_tokens": 47, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-83", + "ok": true, + "error": null, + "completion_tokens": 165, + "reasoning_chars": 728, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-76", + "ok": false, + "error": null, + "completion_tokens": 52, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-212", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-87", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-1", + "ok": true, + "error": null, + "completion_tokens": 1206, + "reasoning_chars": 4589, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-140", + "ok": true, + "error": null, + "completion_tokens": 415, + "reasoning_chars": 1814, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-8", + "ok": true, + "error": null, + "completion_tokens": 392, + "reasoning_chars": 1324, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-138", + "ok": true, + "error": null, + "completion_tokens": 273, + "reasoning_chars": 1134, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-221", + "ok": true, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-136", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-14", + "ok": true, + "error": null, + "completion_tokens": 38, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-0", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-207", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-32", + "ok": true, + "error": null, + "completion_tokens": 573, + "reasoning_chars": 2462, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-83", + "ok": true, + "error": null, + "completion_tokens": 293, + "reasoning_chars": 579, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-220", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-234", + "ok": true, + "error": null, + "completion_tokens": 637, + "reasoning_chars": 1718, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-61", + "ok": true, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-21", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-31", + "ok": true, + "error": null, + "completion_tokens": 189, + "reasoning_chars": 346, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-13", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-211", + "ok": true, + "error": null, + "completion_tokens": 949, + "reasoning_chars": 3730, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-37", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-225", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-90", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 50, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-101", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-179", + "ok": true, + "error": null, + "completion_tokens": 317, + "reasoning_chars": 1056, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-173", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-105", + "ok": true, + "error": null, + "completion_tokens": 164, + "reasoning_chars": 612, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-88", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-68", + "ok": true, + "error": null, + "completion_tokens": 241, + "reasoning_chars": 796, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-185", + "ok": true, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-227", + "ok": true, + "error": null, + "completion_tokens": 134, + "reasoning_chars": 391, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-129", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-238", + "ok": true, + "error": null, + "completion_tokens": 272, + "reasoning_chars": 674, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-202", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-93", + "ok": true, + "error": null, + "completion_tokens": 279, + "reasoning_chars": 1126, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-93", + "ok": true, + "error": null, + "completion_tokens": 1197, + "reasoning_chars": 4710, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-161", + "ok": false, + "error": null, + "completion_tokens": 26, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-46", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-227", + "ok": true, + "error": null, + "completion_tokens": 467, + "reasoning_chars": 1578, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-158", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-205", + "ok": true, + "error": null, + "completion_tokens": 981, + "reasoning_chars": 2483, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-89", + "ok": true, + "error": null, + "completion_tokens": 187, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-177", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-65", + "ok": true, + "error": null, + "completion_tokens": 256, + "reasoning_chars": 682, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-72", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-49", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 410, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-54", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-64", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-86", + "ok": false, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-108", + "ok": true, + "error": null, + "completion_tokens": 89, + "reasoning_chars": 358, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-194", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-194", + "ok": true, + "error": null, + "completion_tokens": 150, + "reasoning_chars": 647, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-75", + "ok": true, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 69171, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-106", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-81", + "ok": true, + "error": null, + "completion_tokens": 208, + "reasoning_chars": 864, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-242", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-150", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-25", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-139", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-12", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 1042, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-178", + "ok": true, + "error": null, + "completion_tokens": 258, + "reasoning_chars": 944, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-160", + "ok": false, + "error": null, + "completion_tokens": 27, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-115", + "ok": true, + "error": null, + "completion_tokens": 11269, + "reasoning_chars": 45590, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_five_objects-44", + "ok": true, + "error": null, + "completion_tokens": 997, + "reasoning_chars": 3708, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-189", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 1307, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-102", + "ok": true, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 1081, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-38", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-141", + "ok": false, + "error": null, + "completion_tokens": 37, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-38", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-168", + "ok": true, + "error": null, + "completion_tokens": 598, + "reasoning_chars": 1396, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-212", + "ok": false, + "error": null, + "completion_tokens": 18650, + "reasoning_chars": 49380, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-245", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-222", + "ok": true, + "error": null, + "completion_tokens": 292, + "reasoning_chars": 1083, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-244", + "ok": true, + "error": null, + "completion_tokens": 318, + "reasoning_chars": 1159, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-35", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-126", + "ok": false, + "error": null, + "completion_tokens": 56, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-90", + "ok": false, + "error": null, + "completion_tokens": 53, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-5", + "ok": true, + "error": null, + "completion_tokens": 238, + "reasoning_chars": 970, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-241", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-234", + "ok": true, + "error": null, + "completion_tokens": 562, + "reasoning_chars": 1388, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-233", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-5", + "ok": false, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-152", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-9", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-209", + "ok": false, + "error": null, + "completion_tokens": 5, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-171", + "ok": true, + "error": null, + "completion_tokens": 540, + "reasoning_chars": 1790, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-navigate-160", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-166", + "ok": false, + "error": null, + "completion_tokens": 159, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-213", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-reasoning_about_colored_objects-240", + "ok": true, + "error": null, + "completion_tokens": 45, + "reasoning_chars": 77, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-95", + "ok": true, + "error": null, + "completion_tokens": 126, + "reasoning_chars": 479, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_five_objects-227", + "ok": true, + "error": null, + "completion_tokens": 511, + "reasoning_chars": 1513, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-96", + "ok": true, + "error": null, + "completion_tokens": 897, + "reasoning_chars": 3880, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-33", + "ok": true, + "error": null, + "completion_tokens": 566, + "reasoning_chars": 102, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-4", + "ok": true, + "error": null, + "completion_tokens": 347, + "reasoning_chars": 1154, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-180", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-74", + "ok": true, + "error": null, + "completion_tokens": 304, + "reasoning_chars": 1217, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-88", + "ok": false, + "error": null, + "completion_tokens": 10, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-10", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-191", + "ok": true, + "error": null, + "completion_tokens": 1805, + "reasoning_chars": 6968, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-71", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-disambiguation_qa-150", + "ok": false, + "error": null, + "completion_tokens": 22, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-188", + "ok": true, + "error": null, + "completion_tokens": 255, + "reasoning_chars": 317, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-191", + "ok": true, + "error": null, + "completion_tokens": 236, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-web_of_lies-203", + "ok": true, + "error": null, + "completion_tokens": 186, + "reasoning_chars": 695, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-237", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 584, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-penguins_in_a_table-24", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-27", + "ok": true, + "error": null, + "completion_tokens": 512, + "reasoning_chars": 1351, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-177", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-boolean_expressions-192", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-242", + "ok": true, + "error": null, + "completion_tokens": 191, + "reasoning_chars": 453, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-dyck_languages-51", + "ok": true, + "error": null, + "completion_tokens": 2429, + "reasoning_chars": 9954, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_three_objects-96", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-word_sorting-100", + "ok": false, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-204", + "ok": false, + "error": null, + "completion_tokens": 7, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_seven_objects-58", + "ok": true, + "error": null, + "completion_tokens": 610, + "reasoning_chars": 1893, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-object_counting-157", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-104", + "ok": true, + "error": null, + "completion_tokens": 35, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-47", + "ok": false, + "error": null, + "completion_tokens": 24, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-date_understanding-150", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-multistep_arithmetic_two-23", + "ok": true, + "error": null, + "completion_tokens": 1116, + "reasoning_chars": 2348, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-sports_understanding-68", + "ok": false, + "error": null, + "completion_tokens": 8, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-174", + "ok": true, + "error": null, + "completion_tokens": 206, + "reasoning_chars": 306, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-tracking_shuffled_objects_seven_objects-108", + "ok": true, + "error": null, + "completion_tokens": 771, + "reasoning_chars": 2620, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-203", + "ok": false, + "error": null, + "completion_tokens": 342, + "reasoning_chars": 615, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-formal_fallacies-115", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-hyperbaton-223", + "ok": true, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-salient_translation_error_detection-221", + "ok": true, + "error": null, + "completion_tokens": 265, + "reasoning_chars": 352, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-temporal_sequences-5", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-geometric_shapes-233", + "ok": true, + "error": null, + "completion_tokens": 253, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-ruin_names-3", + "ok": true, + "error": null, + "completion_tokens": 196, + "reasoning_chars": 708, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-logical_deduction_three_objects-119", + "ok": true, + "error": null, + "completion_tokens": 184, + "reasoning_chars": 588, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-movie_recommendation-197", + "ok": false, + "error": null, + "completion_tokens": 9, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-snarks-103", + "ok": true, + "error": null, + "completion_tokens": 940, + "reasoning_chars": 3978, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "bbh-causal_judgement-178", + "ok": false, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + } + ] + }, + "humaneval": { + "task": "humaneval", + "n": 164, + "hits": 148, + "accuracy_pct": 90.24, + "accuracy_ex_trunc_pct": 91.36, + "truncated": 2, + "completion_tokens_p50": 477, + "completion_tokens_p90": 1716, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 1219, + "reasoning_chars_p90": 5765, + "reasoning_leaked": null, + "empty": 2, + "length_limited": 2, + "errors": 0, + "elapsed_s": 11691.9, + "gen_tokens": 244101, + "tok_per_s": 20.9, + "details": [ + { + "id": "HumanEval/0", + "ok": true, + "error": null, + "completion_tokens": 341, + "reasoning_chars": 1253, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/1", + "ok": false, + "error": null, + "completion_tokens": 998, + "reasoning_chars": 4041, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/2", + "ok": true, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 924, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/3", + "ok": true, + "error": null, + "completion_tokens": 247, + "reasoning_chars": 899, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/4", + "ok": true, + "error": null, + "completion_tokens": 480, + "reasoning_chars": 1839, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/5", + "ok": true, + "error": null, + "completion_tokens": 559, + "reasoning_chars": 2337, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/6", + "ok": true, + "error": null, + "completion_tokens": 409, + "reasoning_chars": 1371, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/7", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 1676, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/8", + "ok": true, + "error": null, + "completion_tokens": 219, + "reasoning_chars": 695, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/9", + "ok": true, + "error": null, + "completion_tokens": 778, + "reasoning_chars": 2979, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/10", + "ok": true, + "error": null, + "completion_tokens": 1524, + "reasoning_chars": 5765, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/11", + "ok": true, + "error": null, + "completion_tokens": 2393, + "reasoning_chars": 8836, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/12", + "ok": false, + "error": null, + "completion_tokens": 31, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/13", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/14", + "ok": true, + "error": null, + "completion_tokens": 215, + "reasoning_chars": 626, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/15", + "ok": true, + "error": null, + "completion_tokens": 320, + "reasoning_chars": 915, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/16", + "ok": true, + "error": null, + "completion_tokens": 21, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/17", + "ok": true, + "error": null, + "completion_tokens": 418, + "reasoning_chars": 804, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/18", + "ok": true, + "error": null, + "completion_tokens": 564, + "reasoning_chars": 1664, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/19", + "ok": true, + "error": null, + "completion_tokens": 461, + "reasoning_chars": 1261, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/20", + "ok": true, + "error": null, + "completion_tokens": 1585, + "reasoning_chars": 6930, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/21", + "ok": true, + "error": null, + "completion_tokens": 1729, + "reasoning_chars": 5940, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/22", + "ok": false, + "error": null, + "completion_tokens": 1950, + "reasoning_chars": 7568, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/23", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 559, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/24", + "ok": true, + "error": null, + "completion_tokens": 542, + "reasoning_chars": 1608, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/25", + "ok": true, + "error": null, + "completion_tokens": 1035, + "reasoning_chars": 2803, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/26", + "ok": false, + "error": null, + "completion_tokens": 813, + "reasoning_chars": 3512, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/27", + "ok": true, + "error": null, + "completion_tokens": 416, + "reasoning_chars": 1443, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/28", + "ok": true, + "error": null, + "completion_tokens": 306, + "reasoning_chars": 1114, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/29", + "ok": true, + "error": null, + "completion_tokens": 213, + "reasoning_chars": 581, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/30", + "ok": true, + "error": null, + "completion_tokens": 229, + "reasoning_chars": 434, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/31", + "ok": true, + "error": null, + "completion_tokens": 462, + "reasoning_chars": 1007, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/32", + "ok": false, + "error": null, + "completion_tokens": 25587, + "reasoning_chars": 81942, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/33", + "ok": true, + "error": null, + "completion_tokens": 489, + "reasoning_chars": 895, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/34", + "ok": true, + "error": null, + "completion_tokens": 171, + "reasoning_chars": 425, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/35", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/36", + "ok": true, + "error": null, + "completion_tokens": 753, + "reasoning_chars": 1783, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/37", + "ok": true, + "error": null, + "completion_tokens": 781, + "reasoning_chars": 2292, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/38", + "ok": false, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/39", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 41897, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "HumanEval/40", + "ok": true, + "error": null, + "completion_tokens": 461, + "reasoning_chars": 873, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/41", + "ok": true, + "error": null, + "completion_tokens": 555, + "reasoning_chars": 1785, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/42", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 439, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/43", + "ok": true, + "error": null, + "completion_tokens": 373, + "reasoning_chars": 772, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/44", + "ok": true, + "error": null, + "completion_tokens": 421, + "reasoning_chars": 948, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/45", + "ok": true, + "error": null, + "completion_tokens": 17, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/46", + "ok": true, + "error": null, + "completion_tokens": 920, + "reasoning_chars": 1681, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/47", + "ok": true, + "error": null, + "completion_tokens": 518, + "reasoning_chars": 1480, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/48", + "ok": true, + "error": null, + "completion_tokens": 140, + "reasoning_chars": 262, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/49", + "ok": true, + "error": null, + "completion_tokens": 23, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/50", + "ok": false, + "error": null, + "completion_tokens": 42, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/51", + "ok": true, + "error": null, + "completion_tokens": 294, + "reasoning_chars": 575, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/52", + "ok": true, + "error": null, + "completion_tokens": 34, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/53", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/54", + "ok": true, + "error": null, + "completion_tokens": 381, + "reasoning_chars": 804, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/55", + "ok": true, + "error": null, + "completion_tokens": 411, + "reasoning_chars": 942, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/56", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 734, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/57", + "ok": true, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/58", + "ok": true, + "error": null, + "completion_tokens": 407, + "reasoning_chars": 1131, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/59", + "ok": true, + "error": null, + "completion_tokens": 552, + "reasoning_chars": 1382, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/60", + "ok": true, + "error": null, + "completion_tokens": 192, + "reasoning_chars": 304, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/61", + "ok": true, + "error": null, + "completion_tokens": 285, + "reasoning_chars": 612, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/62", + "ok": true, + "error": null, + "completion_tokens": 223, + "reasoning_chars": 442, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/63", + "ok": true, + "error": null, + "completion_tokens": 453, + "reasoning_chars": 721, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/64", + "ok": true, + "error": null, + "completion_tokens": 331, + "reasoning_chars": 626, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/65", + "ok": true, + "error": null, + "completion_tokens": 2378, + "reasoning_chars": 8115, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/66", + "ok": true, + "error": null, + "completion_tokens": 275, + "reasoning_chars": 530, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/67", + "ok": true, + "error": null, + "completion_tokens": 293, + "reasoning_chars": 1094, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/68", + "ok": true, + "error": null, + "completion_tokens": 713, + "reasoning_chars": 2394, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/69", + "ok": true, + "error": null, + "completion_tokens": 477, + "reasoning_chars": 1522, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/70", + "ok": true, + "error": null, + "completion_tokens": 773, + "reasoning_chars": 1989, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/71", + "ok": true, + "error": null, + "completion_tokens": 692, + "reasoning_chars": 1707, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/72", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 355, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/73", + "ok": true, + "error": null, + "completion_tokens": 551, + "reasoning_chars": 1278, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/74", + "ok": true, + "error": null, + "completion_tokens": 371, + "reasoning_chars": 546, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/75", + "ok": true, + "error": null, + "completion_tokens": 1028, + "reasoning_chars": 2896, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/76", + "ok": true, + "error": null, + "completion_tokens": 4313, + "reasoning_chars": 11739, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/77", + "ok": true, + "error": null, + "completion_tokens": 891, + "reasoning_chars": 2223, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/78", + "ok": true, + "error": null, + "completion_tokens": 568, + "reasoning_chars": 713, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/79", + "ok": true, + "error": null, + "completion_tokens": 348, + "reasoning_chars": 727, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/80", + "ok": true, + "error": null, + "completion_tokens": 394, + "reasoning_chars": 705, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/81", + "ok": true, + "error": null, + "completion_tokens": 1000, + "reasoning_chars": 2028, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/82", + "ok": true, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 786, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/83", + "ok": true, + "error": null, + "completion_tokens": 866, + "reasoning_chars": 2290, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/84", + "ok": true, + "error": null, + "completion_tokens": 244, + "reasoning_chars": 241, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/85", + "ok": true, + "error": null, + "completion_tokens": 250, + "reasoning_chars": 625, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/86", + "ok": false, + "error": null, + "completion_tokens": 169, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/87", + "ok": true, + "error": null, + "completion_tokens": 386, + "reasoning_chars": 1314, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/88", + "ok": true, + "error": null, + "completion_tokens": 565, + "reasoning_chars": 1158, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/89", + "ok": true, + "error": null, + "completion_tokens": 548, + "reasoning_chars": 1286, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/90", + "ok": true, + "error": null, + "completion_tokens": 423, + "reasoning_chars": 1133, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/91", + "ok": true, + "error": null, + "completion_tokens": 5826, + "reasoning_chars": 21272, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/92", + "ok": true, + "error": null, + "completion_tokens": 528, + "reasoning_chars": 1831, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/93", + "ok": true, + "error": null, + "completion_tokens": 1387, + "reasoning_chars": 4220, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/94", + "ok": true, + "error": null, + "completion_tokens": 1097, + "reasoning_chars": 2695, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/95", + "ok": true, + "error": null, + "completion_tokens": 616, + "reasoning_chars": 1566, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/96", + "ok": true, + "error": null, + "completion_tokens": 1258, + "reasoning_chars": 3316, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/97", + "ok": true, + "error": null, + "completion_tokens": 286, + "reasoning_chars": 520, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/98", + "ok": true, + "error": null, + "completion_tokens": 299, + "reasoning_chars": 594, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/99", + "ok": true, + "error": null, + "completion_tokens": 23382, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/100", + "ok": true, + "error": null, + "completion_tokens": 667, + "reasoning_chars": 1755, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/101", + "ok": true, + "error": null, + "completion_tokens": 352, + "reasoning_chars": 944, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/102", + "ok": false, + "error": null, + "completion_tokens": 46, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/103", + "ok": true, + "error": null, + "completion_tokens": 5435, + "reasoning_chars": 17998, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/104", + "ok": true, + "error": null, + "completion_tokens": 493, + "reasoning_chars": 1290, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/105", + "ok": true, + "error": null, + "completion_tokens": 1267, + "reasoning_chars": 4350, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/106", + "ok": true, + "error": null, + "completion_tokens": 849, + "reasoning_chars": 2039, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/107", + "ok": true, + "error": null, + "completion_tokens": 537, + "reasoning_chars": 728, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/108", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 35297, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true + }, + { + "id": "HumanEval/109", + "ok": true, + "error": null, + "completion_tokens": 608, + "reasoning_chars": 1832, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/110", + "ok": true, + "error": null, + "completion_tokens": 549, + "reasoning_chars": 1111, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/111", + "ok": true, + "error": null, + "completion_tokens": 712, + "reasoning_chars": 1971, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/112", + "ok": true, + "error": null, + "completion_tokens": 376, + "reasoning_chars": 812, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/113", + "ok": true, + "error": null, + "completion_tokens": 2496, + "reasoning_chars": 7733, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/114", + "ok": true, + "error": null, + "completion_tokens": 72, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/115", + "ok": true, + "error": null, + "completion_tokens": 523, + "reasoning_chars": 664, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/116", + "ok": true, + "error": null, + "completion_tokens": 83, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/117", + "ok": true, + "error": null, + "completion_tokens": 732, + "reasoning_chars": 1843, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/118", + "ok": true, + "error": null, + "completion_tokens": 1017, + "reasoning_chars": 2891, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/119", + "ok": true, + "error": null, + "completion_tokens": 727, + "reasoning_chars": 1783, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/120", + "ok": true, + "error": null, + "completion_tokens": 369, + "reasoning_chars": 1274, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/121", + "ok": true, + "error": null, + "completion_tokens": 323, + "reasoning_chars": 565, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/122", + "ok": true, + "error": null, + "completion_tokens": 48, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/123", + "ok": true, + "error": null, + "completion_tokens": 1015, + "reasoning_chars": 2534, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/124", + "ok": true, + "error": null, + "completion_tokens": 237, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/125", + "ok": true, + "error": null, + "completion_tokens": 1240, + "reasoning_chars": 3874, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/126", + "ok": true, + "error": null, + "completion_tokens": 651, + "reasoning_chars": 2188, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/127", + "ok": false, + "error": null, + "completion_tokens": 706, + "reasoning_chars": 2100, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/128", + "ok": true, + "error": null, + "completion_tokens": 469, + "reasoning_chars": 977, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/129", + "ok": true, + "error": null, + "completion_tokens": 1916, + "reasoning_chars": 6406, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/130", + "ok": false, + "error": null, + "completion_tokens": 1018, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/131", + "ok": true, + "error": null, + "completion_tokens": 379, + "reasoning_chars": 931, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/132", + "ok": false, + "error": null, + "completion_tokens": 3759, + "reasoning_chars": 14637, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/133", + "ok": true, + "error": null, + "completion_tokens": 25, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/134", + "ok": true, + "error": null, + "completion_tokens": 1058, + "reasoning_chars": 3744, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/135", + "ok": true, + "error": null, + "completion_tokens": 511, + "reasoning_chars": 1384, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/136", + "ok": true, + "error": null, + "completion_tokens": 463, + "reasoning_chars": 977, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/137", + "ok": true, + "error": null, + "completion_tokens": 825, + "reasoning_chars": 2511, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/138", + "ok": true, + "error": null, + "completion_tokens": 251, + "reasoning_chars": 565, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/139", + "ok": true, + "error": null, + "completion_tokens": 708, + "reasoning_chars": 1977, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/140", + "ok": true, + "error": null, + "completion_tokens": 732, + "reasoning_chars": 2493, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/141", + "ok": true, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/142", + "ok": true, + "error": null, + "completion_tokens": 997, + "reasoning_chars": 2330, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/143", + "ok": true, + "error": null, + "completion_tokens": 568, + "reasoning_chars": 1219, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/144", + "ok": true, + "error": null, + "completion_tokens": 495, + "reasoning_chars": 1582, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/145", + "ok": false, + "error": null, + "completion_tokens": 17983, + "reasoning_chars": 55975, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/146", + "ok": true, + "error": null, + "completion_tokens": 1716, + "reasoning_chars": 4524, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/147", + "ok": true, + "error": null, + "completion_tokens": 3878, + "reasoning_chars": 9333, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/148", + "ok": true, + "error": null, + "completion_tokens": 468, + "reasoning_chars": 627, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/149", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 974, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/150", + "ok": true, + "error": null, + "completion_tokens": 432, + "reasoning_chars": 957, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/151", + "ok": true, + "error": null, + "completion_tokens": 1495, + "reasoning_chars": 4761, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/152", + "ok": true, + "error": null, + "completion_tokens": 339, + "reasoning_chars": 400, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/153", + "ok": true, + "error": null, + "completion_tokens": 734, + "reasoning_chars": 1395, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/154", + "ok": true, + "error": null, + "completion_tokens": 570, + "reasoning_chars": 1914, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/155", + "ok": true, + "error": null, + "completion_tokens": 525, + "reasoning_chars": 1426, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/156", + "ok": true, + "error": null, + "completion_tokens": 425, + "reasoning_chars": 494, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/157", + "ok": true, + "error": null, + "completion_tokens": 43, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/158", + "ok": true, + "error": null, + "completion_tokens": 642, + "reasoning_chars": 1959, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/159", + "ok": true, + "error": null, + "completion_tokens": 453, + "reasoning_chars": 528, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/160", + "ok": false, + "error": null, + "completion_tokens": 112, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/161", + "ok": true, + "error": null, + "completion_tokens": 334, + "reasoning_chars": 779, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/162", + "ok": true, + "error": null, + "completion_tokens": 378, + "reasoning_chars": 1234, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + }, + { + "id": "HumanEval/163", + "ok": true, + "error": null, + "completion_tokens": 1527, + "reasoning_chars": 4222, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false + } + ] + }, + "ifeval": { + "task": "ifeval", + "n": 100, + "hits": 60, + "accuracy_pct": 60.0, + "accuracy_ex_trunc_pct": 60.61, + "truncated": 1, + "completion_tokens_p50": 305, + "completion_tokens_p90": 2225, + "completion_tokens_max": 32768, + "reasoning_chars_p50": 0, + "reasoning_chars_p90": 3925, + "reasoning_leaked": null, + "empty": 1, + "length_limited": 1, + "errors": 0, + "elapsed_s": 8938.4, + "gen_tokens": 140957, + "tok_per_s": 15.8, + "instruction_level_pct": 74.67, + "instructions_checked": 150, + "details": [ + { + "id": "ifeval-153", + "ok": false, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-258", + "ok": true, + "error": null, + "completion_tokens": 62, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-368", + "ok": true, + "error": null, + "completion_tokens": 51, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-85", + "ok": true, + "error": null, + "completion_tokens": 1083, + "reasoning_chars": 3588, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-246", + "ok": true, + "error": null, + "completion_tokens": 313, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-23", + "ok": true, + "error": null, + "completion_tokens": 2, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-372", + "ok": true, + "error": null, + "completion_tokens": 1543, + "reasoning_chars": 3925, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-387", + "ok": true, + "error": null, + "completion_tokens": 844, + "reasoning_chars": 3164, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-138", + "ok": true, + "error": null, + "completion_tokens": 407, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-56", + "ok": true, + "error": null, + "completion_tokens": 266, + "reasoning_chars": 425, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-277", + "ok": false, + "error": null, + "completion_tokens": 2225, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-407", + "ok": false, + "error": null, + "completion_tokens": 371, + "reasoning_chars": 191, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-171", + "ok": true, + "error": null, + "completion_tokens": 67, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-96", + "ok": true, + "error": null, + "completion_tokens": 18199, + "reasoning_chars": 22505, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-134", + "ok": false, + "error": null, + "completion_tokens": 504, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-218", + "ok": true, + "error": null, + "completion_tokens": 1257, + "reasoning_chars": 685, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-330", + "ok": true, + "error": null, + "completion_tokens": 39, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-6", + "ok": false, + "error": null, + "completion_tokens": 101, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-143", + "ok": true, + "error": null, + "completion_tokens": 5379, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-121", + "ok": true, + "error": null, + "completion_tokens": 1412, + "reasoning_chars": 4429, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-219", + "ok": true, + "error": null, + "completion_tokens": 125, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-524", + "ok": true, + "error": null, + "completion_tokens": 886, + "reasoning_chars": 2181, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-206", + "ok": true, + "error": null, + "completion_tokens": 195, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-144", + "ok": true, + "error": null, + "completion_tokens": 446, + "reasoning_chars": 1057, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-447", + "ok": false, + "error": null, + "completion_tokens": 13, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-492", + "ok": false, + "error": null, + "completion_tokens": 1142, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-485", + "ok": false, + "error": null, + "completion_tokens": 549, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-220", + "ok": true, + "error": null, + "completion_tokens": 1074, + "reasoning_chars": 4316, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-412", + "ok": false, + "error": null, + "completion_tokens": 479, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-342", + "ok": false, + "error": null, + "completion_tokens": 738, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-222", + "ok": true, + "error": null, + "completion_tokens": 7003, + "reasoning_chars": 17411, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-131", + "ok": false, + "error": null, + "completion_tokens": 287, + "reasoning_chars": 834, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-210", + "ok": false, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-137", + "ok": true, + "error": null, + "completion_tokens": 193, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-464", + "ok": true, + "error": null, + "completion_tokens": 64, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-503", + "ok": true, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-446", + "ok": true, + "error": null, + "completion_tokens": 322, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-2", + "ok": true, + "error": null, + "completion_tokens": 194, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-525", + "ok": false, + "error": null, + "completion_tokens": 66, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-27", + "ok": false, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-136", + "ok": true, + "error": null, + "completion_tokens": 505, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-286", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-449", + "ok": true, + "error": null, + "completion_tokens": 1163, + "reasoning_chars": 3970, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-55", + "ok": false, + "error": null, + "completion_tokens": 3573, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-93", + "ok": true, + "error": null, + "completion_tokens": 58, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-399", + "ok": true, + "error": null, + "completion_tokens": 19, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-19", + "ok": false, + "error": null, + "completion_tokens": 113, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-229", + "ok": false, + "error": null, + "completion_tokens": 227, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-181", + "ok": true, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-362", + "ok": true, + "error": null, + "completion_tokens": 124, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-289", + "ok": true, + "error": null, + "completion_tokens": 141, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-356", + "ok": true, + "error": null, + "completion_tokens": 245, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-115", + "ok": true, + "error": null, + "completion_tokens": 789, + "reasoning_chars": 2928, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-404", + "ok": false, + "error": null, + "completion_tokens": 547, + "reasoning_chars": 2087, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-243", + "ok": false, + "error": null, + "completion_tokens": 32768, + "reasoning_chars": 96112, + "empty": true, + "finish_reason": "length", + "content_mode": "unclosed-length-limit", + "length_limited": true, + "truncated": true, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-90", + "ok": false, + "error": null, + "completion_tokens": 421, + "reasoning_chars": 490, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-54", + "ok": false, + "error": null, + "completion_tokens": 18, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-276", + "ok": true, + "error": null, + "completion_tokens": 303, + "reasoning_chars": 375, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-308", + "ok": true, + "error": null, + "completion_tokens": 382, + "reasoning_chars": 1056, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-64", + "ok": false, + "error": null, + "completion_tokens": 931, + "reasoning_chars": 3632, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-260", + "ok": true, + "error": null, + "completion_tokens": 14, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-337", + "ok": true, + "error": null, + "completion_tokens": 61, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-510", + "ok": true, + "error": null, + "completion_tokens": 1079, + "reasoning_chars": 3083, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-119", + "ok": true, + "error": null, + "completion_tokens": 1329, + "reasoning_chars": 4119, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-5", + "ok": false, + "error": null, + "completion_tokens": 109, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-468", + "ok": false, + "error": null, + "completion_tokens": 330, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-418", + "ok": false, + "error": null, + "completion_tokens": 399, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-476", + "ok": false, + "error": null, + "completion_tokens": 3108, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-49", + "ok": true, + "error": null, + "completion_tokens": 4773, + "reasoning_chars": 10209, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-336", + "ok": true, + "error": null, + "completion_tokens": 307, + "reasoning_chars": 788, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-422", + "ok": false, + "error": null, + "completion_tokens": 305, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-417", + "ok": true, + "error": null, + "completion_tokens": 107, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-347", + "ok": false, + "error": null, + "completion_tokens": 16311, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-101", + "ok": true, + "error": null, + "completion_tokens": 414, + "reasoning_chars": 66, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-209", + "ok": false, + "error": null, + "completion_tokens": 302, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-341", + "ok": false, + "error": null, + "completion_tokens": 80, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-533", + "ok": false, + "error": null, + "completion_tokens": 59, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 0, + "skipped": false + }, + { + "id": "ifeval-158", + "ok": true, + "error": null, + "completion_tokens": 752, + "reasoning_chars": 2232, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-321", + "ok": true, + "error": null, + "completion_tokens": 802, + "reasoning_chars": 3176, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-311", + "ok": true, + "error": null, + "completion_tokens": 64, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-50", + "ok": false, + "error": null, + "completion_tokens": 79, + "reasoning_chars": 74, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 0, + "passed": 0, + "skipped": true + }, + { + "id": "ifeval-518", + "ok": true, + "error": null, + "completion_tokens": 1762, + "reasoning_chars": 953, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-374", + "ok": true, + "error": null, + "completion_tokens": 863, + "reasoning_chars": 3305, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-317", + "ok": true, + "error": null, + "completion_tokens": 1085, + "reasoning_chars": 4231, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-512", + "ok": true, + "error": null, + "completion_tokens": 57, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-102", + "ok": false, + "error": null, + "completion_tokens": 4, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-72", + "ok": false, + "error": null, + "completion_tokens": 96, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-33", + "ok": false, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-263", + "ok": true, + "error": null, + "completion_tokens": 168, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-501", + "ok": true, + "error": null, + "completion_tokens": 8952, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-4", + "ok": true, + "error": null, + "completion_tokens": 180, + "reasoning_chars": 636, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-444", + "ok": false, + "error": null, + "completion_tokens": 2150, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-228", + "ok": true, + "error": null, + "completion_tokens": 93, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-299", + "ok": true, + "error": null, + "completion_tokens": 3, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-278", + "ok": false, + "error": null, + "completion_tokens": 16, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-154", + "ok": true, + "error": null, + "completion_tokens": 2790, + "reasoning_chars": 10978, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 3, + "skipped": false + }, + { + "id": "ifeval-301", + "ok": true, + "error": null, + "completion_tokens": 652, + "reasoning_chars": 2115, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-161", + "ok": false, + "error": null, + "completion_tokens": 157, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 2, + "passed": 1, + "skipped": false + }, + { + "id": "ifeval-394", + "ok": false, + "error": null, + "completion_tokens": 129, + "reasoning_chars": 0, + "empty": false, + "finish_reason": "stop", + "content_mode": "publisher-direct-content", + "length_limited": false, + "truncated": false, + "checked": 3, + "passed": 2, + "skipped": false + }, + { + "id": "ifeval-159", + "ok": true, + "error": null, + "completion_tokens": 115, + "reasoning_chars": 414, + "empty": false, + "finish_reason": "stop", + "content_mode": "explicit-close", + "length_limited": false, + "truncated": false, + "checked": 1, + "passed": 1, + "skipped": false + } + ] + } + } + } +} \ No newline at end of file