{
  "calibration": {
    "a": 0.32936498522758484,
    "b": 0.13781841099262238,
    "accept_token": 10330,
    "escalate_token": 81082
  },
  "verifier": {
    "val": {
      "all": {
        "n": 1480,
        "answers_wrong": 0.1757,
        "auroc": 0.8839,
        "false_accept_rate": 0.4231,
        "false_escalate_rate": 0.0131,
        "ece": 0.0173,
        "brier": 0.0723
      },
      "uncalibrated_ece": 0.0724,
      "glm-5.3-flash": {
        "n": 740,
        "answers_wrong": 0.123,
        "auroc": 0.8637,
        "false_accept_rate": 0.5604,
        "false_escalate_rate": 0.0046,
        "ece": 0.0223,
        "brier": 0.0629
      },
      "kimi-vl": {
        "n": 370,
        "answers_wrong": 0.3243,
        "auroc": 0.9005,
        "false_accept_rate": 0.2583,
        "false_escalate_rate": 0.04,
        "ece": 0.0632,
        "brier": 0.0928
      },
      "qwen3.8-27b": {
        "n": 370,
        "answers_wrong": 0.1324,
        "auroc": 0.8666,
        "false_accept_rate": 0.5714,
        "false_escalate_rate": 0.0093,
        "ece": 0.0299,
        "brier": 0.0704
      },
      "source:aqua": {
        "n": 212,
        "answers_wrong": 0.184,
        "auroc": 0.8519,
        "false_accept_rate": 0.4359,
        "false_escalate_rate": 0.0058,
        "ece": 0.0353,
        "brier": 0.0735
      },
      "source:arc": {
        "n": 160,
        "answers_wrong": 0.0625,
        "auroc": 0.742,
        "false_accept_rate": 1.0,
        "false_escalate_rate": 0.0067,
        "ece": 0.0419,
        "brier": 0.0556
      },
      "source:csqa": {
        "n": 116,
        "answers_wrong": 0.25,
        "auroc": 0.4935,
        "false_accept_rate": 0.931,
        "false_escalate_rate": 0.023,
        "ece": 0.1432,
        "brier": 0.203
      },
      "source:gsm8k": {
        "n": 264,
        "answers_wrong": 0.0417,
        "auroc": 0.9677,
        "false_accept_rate": 0.2727,
        "false_escalate_rate": 0.004,
        "ece": 0.0499,
        "brier": 0.0147
      },
      "source:humaneval": {
        "n": 36,
        "answers_wrong": 0.0278,
        "auroc": 1.0,
        "false_accept_rate": 0.0,
        "false_escalate_rate": 0.0,
        "ece": 0.0512,
        "brier": 0.0055
      },
      "source:math": {
        "n": 248,
        "answers_wrong": 0.3024,
        "auroc": 0.9272,
        "false_accept_rate": 0.1467,
        "false_escalate_rate": 0.0173,
        "ece": 0.0534,
        "brier": 0.0539
      },
      "source:mbpp": {
        "n": 168,
        "answers_wrong": 0.131,
        "auroc": 0.9692,
        "false_accept_rate": 0.2273,
        "false_escalate_rate": 0.0274,
        "ece": 0.0483,
        "brier": 0.0449
      },
      "source:mmlu_pro": {
        "n": 276,
        "answers_wrong": 0.2645,
        "auroc": 0.8385,
        "false_accept_rate": 0.5068,
        "false_escalate_rate": 0.0197,
        "ece": 0.0517,
        "brier": 0.1229
      },
      "reliability": [
        {
          "bin": "0.0-0.1",
          "n": 104,
          "confidence": 0.029,
          "accuracy": 0.0
        },
        {
          "bin": "0.1-0.2",
          "n": 20,
          "confidence": 0.13,
          "accuracy": 0.1
        },
        {
          "bin": "0.2-0.3",
          "n": 17,
          "confidence": 0.263,
          "accuracy": 0.176
        },
        {
          "bin": "0.3-0.4",
          "n": 11,
          "confidence": 0.355,
          "accuracy": 0.273
        },
        {
          "bin": "0.4-0.5",
          "n": 14,
          "confidence": 0.464,
          "accuracy": 0.571
        },
        {
          "bin": "0.5-0.6",
          "n": 23,
          "confidence": 0.563,
          "accuracy": 0.696
        },
        {
          "bin": "0.6-0.7",
          "n": 36,
          "confidence": 0.66,
          "accuracy": 0.778
        },
        {
          "bin": "0.7-0.8",
          "n": 84,
          "confidence": 0.762,
          "accuracy": 0.81
        },
        {
          "bin": "0.8-0.9",
          "n": 252,
          "confidence": 0.858,
          "accuracy": 0.857
        },
        {
          "bin": "0.9-1.0",
          "n": 919,
          "confidence": 0.96,
          "accuracy": 0.953
        }
      ]
    },
    "test": {
      "all": {
        "n": 2532,
        "answers_wrong": 0.1761,
        "auroc": 0.9195,
        "false_accept_rate": 0.3587,
        "false_escalate_rate": 0.0187,
        "ece": 0.017,
        "brier": 0.0631
      },
      "uncalibrated_ece": 0.0614,
      "glm-5.3-flash": {
        "n": 1266,
        "answers_wrong": 0.1177,
        "auroc": 0.9102,
        "false_accept_rate": 0.4161,
        "false_escalate_rate": 0.0063,
        "ece": 0.0106,
        "brier": 0.0458
      },
      "kimi-vl": {
        "n": 633,
        "answers_wrong": 0.346,
        "auroc": 0.896,
        "false_accept_rate": 0.2694,
        "false_escalate_rate": 0.0507,
        "ece": 0.0418,
        "brier": 0.1013
      },
      "qwen3.8-27b": {
        "n": 633,
        "answers_wrong": 0.1232,
        "auroc": 0.9152,
        "false_accept_rate": 0.5,
        "false_escalate_rate": 0.0198,
        "ece": 0.0248,
        "brier": 0.0595
      },
      "source:aqua": {
        "n": 316,
        "answers_wrong": 0.1677,
        "auroc": 0.8961,
        "false_accept_rate": 0.3019,
        "false_escalate_rate": 0.0304,
        "ece": 0.0288,
        "brier": 0.0627
      },
      "source:arc": {
        "n": 300,
        "answers_wrong": 0.04,
        "auroc": 0.8423,
        "false_accept_rate": 0.9167,
        "false_escalate_rate": 0.0069,
        "ece": 0.0286,
        "brier": 0.0392
      },
      "source:csqa": {
        "n": 252,
        "answers_wrong": 0.1825,
        "auroc": 0.5933,
        "false_accept_rate": 0.8913,
        "false_escalate_rate": 0.0583,
        "ece": 0.087,
        "brier": 0.1548
      },
      "source:gsm8k": {
        "n": 420,
        "answers_wrong": 0.0619,
        "auroc": 0.8544,
        "false_accept_rate": 0.5385,
        "false_escalate_rate": 0.0025,
        "ece": 0.0175,
        "brier": 0.0335
      },
      "source:humaneval": {
        "n": 72,
        "answers_wrong": 0.0972,
        "auroc": 0.9912,
        "false_accept_rate": 0.1429,
        "false_escalate_rate": 0.0,
        "ece": 0.0366,
        "brier": 0.0148
      },
      "source:math": {
        "n": 424,
        "answers_wrong": 0.3184,
        "auroc": 0.9625,
        "false_accept_rate": 0.1407,
        "false_escalate_rate": 0.0035,
        "ece": 0.033,
        "brier": 0.0427
      },
      "source:mbpp": {
        "n": 268,
        "answers_wrong": 0.097,
        "auroc": 0.9545,
        "false_accept_rate": 0.3846,
        "false_escalate_rate": 0.0124,
        "ece": 0.041,
        "brier": 0.0411
      },
      "source:mmlu_pro": {
        "n": 480,
        "answers_wrong": 0.2937,
        "auroc": 0.9275,
        "false_accept_rate": 0.3404,
        "false_escalate_rate": 0.0354,
        "ece": 0.0448,
        "brier": 0.0935
      },
      "reliability": [
        {
          "bin": "0.0-0.1",
          "n": 204,
          "confidence": 0.03,
          "accuracy": 0.005
        },
        {
          "bin": "0.1-0.2",
          "n": 32,
          "confidence": 0.143,
          "accuracy": 0.156
        },
        {
          "bin": "0.2-0.3",
          "n": 40,
          "confidence": 0.248,
          "accuracy": 0.175
        },
        {
          "bin": "0.3-0.4",
          "n": 23,
          "confidence": 0.359,
          "accuracy": 0.478
        },
        {
          "bin": "0.4-0.5",
          "n": 26,
          "confidence": 0.454,
          "accuracy": 0.577
        },
        {
          "bin": "0.5-0.6",
          "n": 33,
          "confidence": 0.553,
          "accuracy": 0.636
        },
        {
          "bin": "0.6-0.7",
          "n": 71,
          "confidence": 0.657,
          "accuracy": 0.746
        },
        {
          "bin": "0.7-0.8",
          "n": 153,
          "confidence": 0.758,
          "accuracy": 0.771
        },
        {
          "bin": "0.8-0.9",
          "n": 360,
          "confidence": 0.86,
          "accuracy": 0.878
        },
        {
          "bin": "0.9-1.0",
          "n": 1590,
          "confidence": 0.961,
          "accuracy": 0.968
        }
      ]
    },
    "ood": {
      "all": {
        "n": 3240,
        "answers_wrong": 0.1877,
        "auroc": 0.72,
        "false_accept_rate": 0.7023,
        "false_escalate_rate": 0.0672,
        "ece": 0.0602,
        "brier": 0.1398
      },
      "uncalibrated_ece": 0.1358,
      "glm-5.3-flash": {
        "n": 1620,
        "answers_wrong": 0.0963,
        "auroc": 0.6469,
        "false_accept_rate": 0.9487,
        "false_escalate_rate": 0.0342,
        "ece": 0.0679,
        "brier": 0.1001
      },
      "kimi-vl": {
        "n": 810,
        "answers_wrong": 0.4642,
        "auroc": 0.6174,
        "false_accept_rate": 0.5851,
        "false_escalate_rate": 0.1797,
        "ece": 0.119,
        "brier": 0.2476
      },
      "qwen3.8-27b": {
        "n": 810,
        "answers_wrong": 0.0938,
        "auroc": 0.6742,
        "false_accept_rate": 0.7763,
        "false_escalate_rate": 0.0668,
        "ece": 0.1386,
        "brier": 0.1115
      },
      "source:bbh": {
        "n": 3240,
        "answers_wrong": 0.1877,
        "auroc": 0.72,
        "false_accept_rate": 0.7023,
        "false_escalate_rate": 0.0672,
        "ece": 0.0602,
        "brier": 0.1398
      },
      "reliability": [
        {
          "bin": "0.0-0.1",
          "n": 30,
          "confidence": 0.066,
          "accuracy": 0.033
        },
        {
          "bin": "0.1-0.2",
          "n": 56,
          "confidence": 0.149,
          "accuracy": 0.125
        },
        {
          "bin": "0.2-0.3",
          "n": 73,
          "confidence": 0.254,
          "accuracy": 0.589
        },
        {
          "bin": "0.3-0.4",
          "n": 82,
          "confidence": 0.357,
          "accuracy": 0.598
        },
        {
          "bin": "0.4-0.5",
          "n": 117,
          "confidence": 0.449,
          "accuracy": 0.658
        },
        {
          "bin": "0.5-0.6",
          "n": 199,
          "confidence": 0.553,
          "accuracy": 0.734
        },
        {
          "bin": "0.6-0.7",
          "n": 375,
          "confidence": 0.654,
          "accuracy": 0.773
        },
        {
          "bin": "0.7-0.8",
          "n": 525,
          "confidence": 0.755,
          "accuracy": 0.794
        },
        {
          "bin": "0.8-0.9",
          "n": 906,
          "confidence": 0.854,
          "accuracy": 0.87
        },
        {
          "bin": "0.9-1.0",
          "n": 877,
          "confidence": 0.939,
          "accuracy": 0.928
        }
      ]
    }
  },
  "ladder": [
    "glm-5.3-flash",
    "qwen3.8-27b",
    "nemotron-ultra"
  ],
  "n_ladder_tasks": 491,
  "policies": {
    "always:glm-5.3-flash": {
      "accuracy": 0.888,
      "relative_cost": 0.1146,
      "silent_failure_rate": 0.0,
      "calls_per_task": 1.0,
      "extra_agent_latency_s": 0.0
    },
    "always:qwen3.8-27b": {
      "accuracy": 0.8737,
      "relative_cost": 0.4208,
      "silent_failure_rate": 0.0,
      "calls_per_task": 1.0,
      "extra_agent_latency_s": 0.0
    },
    "always:nemotron-ultra": {
      "accuracy": 0.8473,
      "relative_cost": 1.0,
      "silent_failure_rate": 0.0,
      "calls_per_task": 1.0,
      "extra_agent_latency_s": 0.0
    },
    "ladder:format-check": {
      "accuracy": 0.8982,
      "relative_cost": 0.4058,
      "silent_failure_rate": 0.0733,
      "calls_per_task": 1.073,
      "extra_agent_latency_s": 10.95
    },
    "ladder:self-consistency": {
      "accuracy": 0.9043,
      "relative_cost": 0.5535,
      "silent_failure_rate": 0.0652,
      "calls_per_task": 1.141,
      "extra_agent_latency_s": 15.29
    },
    "ladder:oracle": {
      "accuracy": 0.9328,
      "relative_cost": 0.473,
      "silent_failure_rate": 0.0,
      "calls_per_task": 1.185,
      "extra_agent_latency_s": 15.89
    },
    "shaktimaan-v1.0": {
      "accuracy": 0.8493,
      "relative_cost": 0.3226
    }
  },
  "jedi_curve": [
    {
      "threshold": 0.05,
      "accuracy": 0.9104,
      "relative_cost": 0.2441,
      "silent_failure_rate": 0.0754,
      "calls_per_task": 1.086,
      "extra_agent_latency_s": 8.94
    },
    {
      "threshold": 0.1,
      "accuracy": 0.9145,
      "relative_cost": 0.2634,
      "silent_failure_rate": 0.0652,
      "calls_per_task": 1.096,
      "extra_agent_latency_s": 9.49
    },
    {
      "threshold": 0.2,
      "accuracy": 0.9124,
      "relative_cost": 0.2746,
      "silent_failure_rate": 0.0652,
      "calls_per_task": 1.098,
      "extra_agent_latency_s": 9.69
    },
    {
      "threshold": 0.3,
      "accuracy": 0.9145,
      "relative_cost": 0.3229,
      "silent_failure_rate": 0.055,
      "calls_per_task": 1.108,
      "extra_agent_latency_s": 10.73
    },
    {
      "threshold": 0.4,
      "accuracy": 0.9145,
      "relative_cost": 0.3551,
      "silent_failure_rate": 0.0509,
      "calls_per_task": 1.114,
      "extra_agent_latency_s": 11.42
    },
    {
      "threshold": 0.5,
      "accuracy": 0.9165,
      "relative_cost": 0.3865,
      "silent_failure_rate": 0.0448,
      "calls_per_task": 1.122,
      "extra_agent_latency_s": 12.08
    },
    {
      "threshold": 0.6,
      "accuracy": 0.9165,
      "relative_cost": 0.4313,
      "silent_failure_rate": 0.0387,
      "calls_per_task": 1.13,
      "extra_agent_latency_s": 13.14
    },
    {
      "threshold": 0.7,
      "accuracy": 0.9185,
      "relative_cost": 0.4538,
      "silent_failure_rate": 0.0285,
      "calls_per_task": 1.147,
      "extra_agent_latency_s": 14.32
    },
    {
      "threshold": 0.8,
      "accuracy": 0.9104,
      "relative_cost": 0.5055,
      "silent_failure_rate": 0.0285,
      "calls_per_task": 1.196,
      "extra_agent_latency_s": 16.37
    },
    {
      "threshold": 0.85,
      "accuracy": 0.9084,
      "relative_cost": 0.5408,
      "silent_failure_rate": 0.0224,
      "calls_per_task": 1.238,
      "extra_agent_latency_s": 17.9
    },
    {
      "threshold": 0.9,
      "accuracy": 0.9022,
      "relative_cost": 0.6093,
      "silent_failure_rate": 0.0163,
      "calls_per_task": 1.316,
      "extra_agent_latency_s": 20.84
    },
    {
      "threshold": 0.95,
      "accuracy": 0.8697,
      "relative_cost": 0.8913,
      "silent_failure_rate": 0.0143,
      "calls_per_task": 1.654,
      "extra_agent_latency_s": 34.46
    },
    {
      "threshold": 0.98,
      "accuracy": 0.8615,
      "relative_cost": 1.2413,
      "silent_failure_rate": 0.0,
      "calls_per_task": 2.393,
      "extra_agent_latency_s": 55.17
    }
  ],
  "two_rung": {
    "new-domains (aqua, csqa)": {
      "n_tasks": 142,
      "always:glm-5.3-flash": {
        "accuracy": 0.8521,
        "cost_vs_always_mid": 0.2999,
        "silent_failure_rate": 0.0
      },
      "always:qwen3.8-27b": {
        "accuracy": 0.8873,
        "cost_vs_always_mid": 1.0,
        "silent_failure_rate": 0.0
      },
      "oracle": {
        "accuracy": 0.9225,
        "cost_vs_always_mid": 0.7111,
        "silent_failure_rate": 0.0
      },
      "self-consistency": {
        "accuracy": 0.8873,
        "cost_vs_always_mid": 1.0475,
        "silent_failure_rate": 0.0704
      },
      "jedi@0.5": {
        "accuracy": 0.8873,
        "cost_vs_always_mid": 0.6761,
        "silent_failure_rate": 0.0845
      },
      "jedi@0.7": {
        "accuracy": 0.8732,
        "cost_vs_always_mid": 0.7191,
        "silent_failure_rate": 0.0845
      },
      "jedi@0.9": {
        "accuracy": 0.8873,
        "cost_vs_always_mid": 0.8953,
        "silent_failure_rate": 0.0493
      }
    },
    "ood: bbh": {
      "n_tasks": 810,
      "always:glm-5.3-flash": {
        "accuracy": 0.9012,
        "cost_vs_always_mid": 0.1436,
        "silent_failure_rate": 0.0
      },
      "always:qwen3.8-27b": {
        "accuracy": 0.9062,
        "cost_vs_always_mid": 1.0,
        "silent_failure_rate": 0.0
      },
      "oracle": {
        "accuracy": 0.9457,
        "cost_vs_always_mid": 0.3743,
        "silent_failure_rate": 0.0
      },
      "self-consistency": {
        "accuracy": 0.9136,
        "cost_vs_always_mid": 0.5322,
        "silent_failure_rate": 0.0605
      },
      "jedi@0.5": {
        "accuracy": 0.9049,
        "cost_vs_always_mid": 0.178,
        "silent_failure_rate": 0.0938
      },
      "jedi@0.7": {
        "accuracy": 0.9111,
        "cost_vs_always_mid": 0.2665,
        "silent_failure_rate": 0.0778
      },
      "jedi@0.9": {
        "accuracy": 0.9198,
        "cost_vs_always_mid": 0.6854,
        "silent_failure_rate": 0.0222
      }
    }
  },
  "overhead": {
    "jedi_ms_per_check_batch1": 55.6,
    "jedi_input_tokens_mean": 451,
    "jedi_output_tokens": 1
  }
}