| { |
| "calibration": { |
| "a": 0.32936498522758484, |
| "b": 0.13781841099262238, |
| "accept_token": 10330, |
| "escalate_token": 81082 |
| }, |
| "verifier": { |
| "val": { |
| "all": { |
| "n": 1480, |
| "answers_wrong": 0.1757, |
| "auroc": 0.8839, |
| "false_accept_rate": 0.4231, |
| "false_escalate_rate": 0.0131, |
| "ece": 0.0173, |
| "brier": 0.0723 |
| }, |
| "uncalibrated_ece": 0.0724, |
| "glm-5.3-flash": { |
| "n": 740, |
| "answers_wrong": 0.123, |
| "auroc": 0.8637, |
| "false_accept_rate": 0.5604, |
| "false_escalate_rate": 0.0046, |
| "ece": 0.0223, |
| "brier": 0.0629 |
| }, |
| "kimi-vl": { |
| "n": 370, |
| "answers_wrong": 0.3243, |
| "auroc": 0.9005, |
| "false_accept_rate": 0.2583, |
| "false_escalate_rate": 0.04, |
| "ece": 0.0632, |
| "brier": 0.0928 |
| }, |
| "qwen3.8-27b": { |
| "n": 370, |
| "answers_wrong": 0.1324, |
| "auroc": 0.8666, |
| "false_accept_rate": 0.5714, |
| "false_escalate_rate": 0.0093, |
| "ece": 0.0299, |
| "brier": 0.0704 |
| }, |
| "source:aqua": { |
| "n": 212, |
| "answers_wrong": 0.184, |
| "auroc": 0.8519, |
| "false_accept_rate": 0.4359, |
| "false_escalate_rate": 0.0058, |
| "ece": 0.0353, |
| "brier": 0.0735 |
| }, |
| "source:arc": { |
| "n": 160, |
| "answers_wrong": 0.0625, |
| "auroc": 0.742, |
| "false_accept_rate": 1.0, |
| "false_escalate_rate": 0.0067, |
| "ece": 0.0419, |
| "brier": 0.0556 |
| }, |
| "source:csqa": { |
| "n": 116, |
| "answers_wrong": 0.25, |
| "auroc": 0.4935, |
| "false_accept_rate": 0.931, |
| "false_escalate_rate": 0.023, |
| "ece": 0.1432, |
| "brier": 0.203 |
| }, |
| "source:gsm8k": { |
| "n": 264, |
| "answers_wrong": 0.0417, |
| "auroc": 0.9677, |
| "false_accept_rate": 0.2727, |
| "false_escalate_rate": 0.004, |
| "ece": 0.0499, |
| "brier": 0.0147 |
| }, |
| "source:humaneval": { |
| "n": 36, |
| "answers_wrong": 0.0278, |
| "auroc": 1.0, |
| "false_accept_rate": 0.0, |
| "false_escalate_rate": 0.0, |
| "ece": 0.0512, |
| "brier": 0.0055 |
| }, |
| "source:math": { |
| "n": 248, |
| "answers_wrong": 0.3024, |
| "auroc": 0.9272, |
| "false_accept_rate": 0.1467, |
| "false_escalate_rate": 0.0173, |
| "ece": 0.0534, |
| "brier": 0.0539 |
| }, |
| "source:mbpp": { |
| "n": 168, |
| "answers_wrong": 0.131, |
| "auroc": 0.9692, |
| "false_accept_rate": 0.2273, |
| "false_escalate_rate": 0.0274, |
| "ece": 0.0483, |
| "brier": 0.0449 |
| }, |
| "source:mmlu_pro": { |
| "n": 276, |
| "answers_wrong": 0.2645, |
| "auroc": 0.8385, |
| "false_accept_rate": 0.5068, |
| "false_escalate_rate": 0.0197, |
| "ece": 0.0517, |
| "brier": 0.1229 |
| }, |
| "reliability": [ |
| { |
| "bin": "0.0-0.1", |
| "n": 104, |
| "confidence": 0.029, |
| "accuracy": 0.0 |
| }, |
| { |
| "bin": "0.1-0.2", |
| "n": 20, |
| "confidence": 0.13, |
| "accuracy": 0.1 |
| }, |
| { |
| "bin": "0.2-0.3", |
| "n": 17, |
| "confidence": 0.263, |
| "accuracy": 0.176 |
| }, |
| { |
| "bin": "0.3-0.4", |
| "n": 11, |
| "confidence": 0.355, |
| "accuracy": 0.273 |
| }, |
| { |
| "bin": "0.4-0.5", |
| "n": 14, |
| "confidence": 0.464, |
| "accuracy": 0.571 |
| }, |
| { |
| "bin": "0.5-0.6", |
| "n": 23, |
| "confidence": 0.563, |
| "accuracy": 0.696 |
| }, |
| { |
| "bin": "0.6-0.7", |
| "n": 36, |
| "confidence": 0.66, |
| "accuracy": 0.778 |
| }, |
| { |
| "bin": "0.7-0.8", |
| "n": 84, |
| "confidence": 0.762, |
| "accuracy": 0.81 |
| }, |
| { |
| "bin": "0.8-0.9", |
| "n": 252, |
| "confidence": 0.858, |
| "accuracy": 0.857 |
| }, |
| { |
| "bin": "0.9-1.0", |
| "n": 919, |
| "confidence": 0.96, |
| "accuracy": 0.953 |
| } |
| ] |
| }, |
| "test": { |
| "all": { |
| "n": 2532, |
| "answers_wrong": 0.1761, |
| "auroc": 0.9195, |
| "false_accept_rate": 0.3587, |
| "false_escalate_rate": 0.0187, |
| "ece": 0.017, |
| "brier": 0.0631 |
| }, |
| "uncalibrated_ece": 0.0614, |
| "glm-5.3-flash": { |
| "n": 1266, |
| "answers_wrong": 0.1177, |
| "auroc": 0.9102, |
| "false_accept_rate": 0.4161, |
| "false_escalate_rate": 0.0063, |
| "ece": 0.0106, |
| "brier": 0.0458 |
| }, |
| "kimi-vl": { |
| "n": 633, |
| "answers_wrong": 0.346, |
| "auroc": 0.896, |
| "false_accept_rate": 0.2694, |
| "false_escalate_rate": 0.0507, |
| "ece": 0.0418, |
| "brier": 0.1013 |
| }, |
| "qwen3.8-27b": { |
| "n": 633, |
| "answers_wrong": 0.1232, |
| "auroc": 0.9152, |
| "false_accept_rate": 0.5, |
| "false_escalate_rate": 0.0198, |
| "ece": 0.0248, |
| "brier": 0.0595 |
| }, |
| "source:aqua": { |
| "n": 316, |
| "answers_wrong": 0.1677, |
| "auroc": 0.8961, |
| "false_accept_rate": 0.3019, |
| "false_escalate_rate": 0.0304, |
| "ece": 0.0288, |
| "brier": 0.0627 |
| }, |
| "source:arc": { |
| "n": 300, |
| "answers_wrong": 0.04, |
| "auroc": 0.8423, |
| "false_accept_rate": 0.9167, |
| "false_escalate_rate": 0.0069, |
| "ece": 0.0286, |
| "brier": 0.0392 |
| }, |
| "source:csqa": { |
| "n": 252, |
| "answers_wrong": 0.1825, |
| "auroc": 0.5933, |
| "false_accept_rate": 0.8913, |
| "false_escalate_rate": 0.0583, |
| "ece": 0.087, |
| "brier": 0.1548 |
| }, |
| "source:gsm8k": { |
| "n": 420, |
| "answers_wrong": 0.0619, |
| "auroc": 0.8544, |
| "false_accept_rate": 0.5385, |
| "false_escalate_rate": 0.0025, |
| "ece": 0.0175, |
| "brier": 0.0335 |
| }, |
| "source:humaneval": { |
| "n": 72, |
| "answers_wrong": 0.0972, |
| "auroc": 0.9912, |
| "false_accept_rate": 0.1429, |
| "false_escalate_rate": 0.0, |
| "ece": 0.0366, |
| "brier": 0.0148 |
| }, |
| "source:math": { |
| "n": 424, |
| "answers_wrong": 0.3184, |
| "auroc": 0.9625, |
| "false_accept_rate": 0.1407, |
| "false_escalate_rate": 0.0035, |
| "ece": 0.033, |
| "brier": 0.0427 |
| }, |
| "source:mbpp": { |
| "n": 268, |
| "answers_wrong": 0.097, |
| "auroc": 0.9545, |
| "false_accept_rate": 0.3846, |
| "false_escalate_rate": 0.0124, |
| "ece": 0.041, |
| "brier": 0.0411 |
| }, |
| "source:mmlu_pro": { |
| "n": 480, |
| "answers_wrong": 0.2937, |
| "auroc": 0.9275, |
| "false_accept_rate": 0.3404, |
| "false_escalate_rate": 0.0354, |
| "ece": 0.0448, |
| "brier": 0.0935 |
| }, |
| "reliability": [ |
| { |
| "bin": "0.0-0.1", |
| "n": 204, |
| "confidence": 0.03, |
| "accuracy": 0.005 |
| }, |
| { |
| "bin": "0.1-0.2", |
| "n": 32, |
| "confidence": 0.143, |
| "accuracy": 0.156 |
| }, |
| { |
| "bin": "0.2-0.3", |
| "n": 40, |
| "confidence": 0.248, |
| "accuracy": 0.175 |
| }, |
| { |
| "bin": "0.3-0.4", |
| "n": 23, |
| "confidence": 0.359, |
| "accuracy": 0.478 |
| }, |
| { |
| "bin": "0.4-0.5", |
| "n": 26, |
| "confidence": 0.454, |
| "accuracy": 0.577 |
| }, |
| { |
| "bin": "0.5-0.6", |
| "n": 33, |
| "confidence": 0.553, |
| "accuracy": 0.636 |
| }, |
| { |
| "bin": "0.6-0.7", |
| "n": 71, |
| "confidence": 0.657, |
| "accuracy": 0.746 |
| }, |
| { |
| "bin": "0.7-0.8", |
| "n": 153, |
| "confidence": 0.758, |
| "accuracy": 0.771 |
| }, |
| { |
| "bin": "0.8-0.9", |
| "n": 360, |
| "confidence": 0.86, |
| "accuracy": 0.878 |
| }, |
| { |
| "bin": "0.9-1.0", |
| "n": 1590, |
| "confidence": 0.961, |
| "accuracy": 0.968 |
| } |
| ] |
| }, |
| "ood": { |
| "all": { |
| "n": 3240, |
| "answers_wrong": 0.1877, |
| "auroc": 0.72, |
| "false_accept_rate": 0.7023, |
| "false_escalate_rate": 0.0672, |
| "ece": 0.0602, |
| "brier": 0.1398 |
| }, |
| "uncalibrated_ece": 0.1358, |
| "glm-5.3-flash": { |
| "n": 1620, |
| "answers_wrong": 0.0963, |
| "auroc": 0.6469, |
| "false_accept_rate": 0.9487, |
| "false_escalate_rate": 0.0342, |
| "ece": 0.0679, |
| "brier": 0.1001 |
| }, |
| "kimi-vl": { |
| "n": 810, |
| "answers_wrong": 0.4642, |
| "auroc": 0.6174, |
| "false_accept_rate": 0.5851, |
| "false_escalate_rate": 0.1797, |
| "ece": 0.119, |
| "brier": 0.2476 |
| }, |
| "qwen3.8-27b": { |
| "n": 810, |
| "answers_wrong": 0.0938, |
| "auroc": 0.6742, |
| "false_accept_rate": 0.7763, |
| "false_escalate_rate": 0.0668, |
| "ece": 0.1386, |
| "brier": 0.1115 |
| }, |
| "source:bbh": { |
| "n": 3240, |
| "answers_wrong": 0.1877, |
| "auroc": 0.72, |
| "false_accept_rate": 0.7023, |
| "false_escalate_rate": 0.0672, |
| "ece": 0.0602, |
| "brier": 0.1398 |
| }, |
| "reliability": [ |
| { |
| "bin": "0.0-0.1", |
| "n": 30, |
| "confidence": 0.066, |
| "accuracy": 0.033 |
| }, |
| { |
| "bin": "0.1-0.2", |
| "n": 56, |
| "confidence": 0.149, |
| "accuracy": 0.125 |
| }, |
| { |
| "bin": "0.2-0.3", |
| "n": 73, |
| "confidence": 0.254, |
| "accuracy": 0.589 |
| }, |
| { |
| "bin": "0.3-0.4", |
| "n": 82, |
| "confidence": 0.357, |
| "accuracy": 0.598 |
| }, |
| { |
| "bin": "0.4-0.5", |
| "n": 117, |
| "confidence": 0.449, |
| "accuracy": 0.658 |
| }, |
| { |
| "bin": "0.5-0.6", |
| "n": 199, |
| "confidence": 0.553, |
| "accuracy": 0.734 |
| }, |
| { |
| "bin": "0.6-0.7", |
| "n": 375, |
| "confidence": 0.654, |
| "accuracy": 0.773 |
| }, |
| { |
| "bin": "0.7-0.8", |
| "n": 525, |
| "confidence": 0.755, |
| "accuracy": 0.794 |
| }, |
| { |
| "bin": "0.8-0.9", |
| "n": 906, |
| "confidence": 0.854, |
| "accuracy": 0.87 |
| }, |
| { |
| "bin": "0.9-1.0", |
| "n": 877, |
| "confidence": 0.939, |
| "accuracy": 0.928 |
| } |
| ] |
| } |
| }, |
| "ladder": [ |
| "glm-5.3-flash", |
| "qwen3.8-27b", |
| "nemotron-ultra" |
| ], |
| "n_ladder_tasks": 491, |
| "policies": { |
| "always:glm-5.3-flash": { |
| "accuracy": 0.888, |
| "relative_cost": 0.1146, |
| "silent_failure_rate": 0.0, |
| "calls_per_task": 1.0, |
| "extra_agent_latency_s": 0.0 |
| }, |
| "always:qwen3.8-27b": { |
| "accuracy": 0.8737, |
| "relative_cost": 0.4208, |
| "silent_failure_rate": 0.0, |
| "calls_per_task": 1.0, |
| "extra_agent_latency_s": 0.0 |
| }, |
| "always:nemotron-ultra": { |
| "accuracy": 0.8473, |
| "relative_cost": 1.0, |
| "silent_failure_rate": 0.0, |
| "calls_per_task": 1.0, |
| "extra_agent_latency_s": 0.0 |
| }, |
| "ladder:format-check": { |
| "accuracy": 0.8982, |
| "relative_cost": 0.4058, |
| "silent_failure_rate": 0.0733, |
| "calls_per_task": 1.073, |
| "extra_agent_latency_s": 10.95 |
| }, |
| "ladder:self-consistency": { |
| "accuracy": 0.9043, |
| "relative_cost": 0.5535, |
| "silent_failure_rate": 0.0652, |
| "calls_per_task": 1.141, |
| "extra_agent_latency_s": 15.29 |
| }, |
| "ladder:oracle": { |
| "accuracy": 0.9328, |
| "relative_cost": 0.473, |
| "silent_failure_rate": 0.0, |
| "calls_per_task": 1.185, |
| "extra_agent_latency_s": 15.89 |
| }, |
| "shaktimaan-v1.0": { |
| "accuracy": 0.8493, |
| "relative_cost": 0.3226 |
| } |
| }, |
| "jedi_curve": [ |
| { |
| "threshold": 0.05, |
| "accuracy": 0.9104, |
| "relative_cost": 0.2441, |
| "silent_failure_rate": 0.0754, |
| "calls_per_task": 1.086, |
| "extra_agent_latency_s": 8.94 |
| }, |
| { |
| "threshold": 0.1, |
| "accuracy": 0.9145, |
| "relative_cost": 0.2634, |
| "silent_failure_rate": 0.0652, |
| "calls_per_task": 1.096, |
| "extra_agent_latency_s": 9.49 |
| }, |
| { |
| "threshold": 0.2, |
| "accuracy": 0.9124, |
| "relative_cost": 0.2746, |
| "silent_failure_rate": 0.0652, |
| "calls_per_task": 1.098, |
| "extra_agent_latency_s": 9.69 |
| }, |
| { |
| "threshold": 0.3, |
| "accuracy": 0.9145, |
| "relative_cost": 0.3229, |
| "silent_failure_rate": 0.055, |
| "calls_per_task": 1.108, |
| "extra_agent_latency_s": 10.73 |
| }, |
| { |
| "threshold": 0.4, |
| "accuracy": 0.9145, |
| "relative_cost": 0.3551, |
| "silent_failure_rate": 0.0509, |
| "calls_per_task": 1.114, |
| "extra_agent_latency_s": 11.42 |
| }, |
| { |
| "threshold": 0.5, |
| "accuracy": 0.9165, |
| "relative_cost": 0.3865, |
| "silent_failure_rate": 0.0448, |
| "calls_per_task": 1.122, |
| "extra_agent_latency_s": 12.08 |
| }, |
| { |
| "threshold": 0.6, |
| "accuracy": 0.9165, |
| "relative_cost": 0.4313, |
| "silent_failure_rate": 0.0387, |
| "calls_per_task": 1.13, |
| "extra_agent_latency_s": 13.14 |
| }, |
| { |
| "threshold": 0.7, |
| "accuracy": 0.9185, |
| "relative_cost": 0.4538, |
| "silent_failure_rate": 0.0285, |
| "calls_per_task": 1.147, |
| "extra_agent_latency_s": 14.32 |
| }, |
| { |
| "threshold": 0.8, |
| "accuracy": 0.9104, |
| "relative_cost": 0.5055, |
| "silent_failure_rate": 0.0285, |
| "calls_per_task": 1.196, |
| "extra_agent_latency_s": 16.37 |
| }, |
| { |
| "threshold": 0.85, |
| "accuracy": 0.9084, |
| "relative_cost": 0.5408, |
| "silent_failure_rate": 0.0224, |
| "calls_per_task": 1.238, |
| "extra_agent_latency_s": 17.9 |
| }, |
| { |
| "threshold": 0.9, |
| "accuracy": 0.9022, |
| "relative_cost": 0.6093, |
| "silent_failure_rate": 0.0163, |
| "calls_per_task": 1.316, |
| "extra_agent_latency_s": 20.84 |
| }, |
| { |
| "threshold": 0.95, |
| "accuracy": 0.8697, |
| "relative_cost": 0.8913, |
| "silent_failure_rate": 0.0143, |
| "calls_per_task": 1.654, |
| "extra_agent_latency_s": 34.46 |
| }, |
| { |
| "threshold": 0.98, |
| "accuracy": 0.8615, |
| "relative_cost": 1.2413, |
| "silent_failure_rate": 0.0, |
| "calls_per_task": 2.393, |
| "extra_agent_latency_s": 55.17 |
| } |
| ], |
| "two_rung": { |
| "new-domains (aqua, csqa)": { |
| "n_tasks": 142, |
| "always:glm-5.3-flash": { |
| "accuracy": 0.8521, |
| "cost_vs_always_mid": 0.2999, |
| "silent_failure_rate": 0.0 |
| }, |
| "always:qwen3.8-27b": { |
| "accuracy": 0.8873, |
| "cost_vs_always_mid": 1.0, |
| "silent_failure_rate": 0.0 |
| }, |
| "oracle": { |
| "accuracy": 0.9225, |
| "cost_vs_always_mid": 0.7111, |
| "silent_failure_rate": 0.0 |
| }, |
| "self-consistency": { |
| "accuracy": 0.8873, |
| "cost_vs_always_mid": 1.0475, |
| "silent_failure_rate": 0.0704 |
| }, |
| "jedi@0.5": { |
| "accuracy": 0.8873, |
| "cost_vs_always_mid": 0.6761, |
| "silent_failure_rate": 0.0845 |
| }, |
| "jedi@0.7": { |
| "accuracy": 0.8732, |
| "cost_vs_always_mid": 0.7191, |
| "silent_failure_rate": 0.0845 |
| }, |
| "jedi@0.9": { |
| "accuracy": 0.8873, |
| "cost_vs_always_mid": 0.8953, |
| "silent_failure_rate": 0.0493 |
| } |
| }, |
| "ood: bbh": { |
| "n_tasks": 810, |
| "always:glm-5.3-flash": { |
| "accuracy": 0.9012, |
| "cost_vs_always_mid": 0.1436, |
| "silent_failure_rate": 0.0 |
| }, |
| "always:qwen3.8-27b": { |
| "accuracy": 0.9062, |
| "cost_vs_always_mid": 1.0, |
| "silent_failure_rate": 0.0 |
| }, |
| "oracle": { |
| "accuracy": 0.9457, |
| "cost_vs_always_mid": 0.3743, |
| "silent_failure_rate": 0.0 |
| }, |
| "self-consistency": { |
| "accuracy": 0.9136, |
| "cost_vs_always_mid": 0.5322, |
| "silent_failure_rate": 0.0605 |
| }, |
| "jedi@0.5": { |
| "accuracy": 0.9049, |
| "cost_vs_always_mid": 0.178, |
| "silent_failure_rate": 0.0938 |
| }, |
| "jedi@0.7": { |
| "accuracy": 0.9111, |
| "cost_vs_always_mid": 0.2665, |
| "silent_failure_rate": 0.0778 |
| }, |
| "jedi@0.9": { |
| "accuracy": 0.9198, |
| "cost_vs_always_mid": 0.6854, |
| "silent_failure_rate": 0.0222 |
| } |
| } |
| }, |
| "overhead": { |
| "jedi_ms_per_check_batch1": 55.6, |
| "jedi_input_tokens_mean": 451, |
| "jedi_output_tokens": 1 |
| } |
| } |
