{
 "n_test": 13632,
 "n_calib": 5572,
 "traits": {
  "A": {
   "n": 1134,
   "category_acc": 0.8854,
   "category_macro_f1": 0.8722,
   "needs_tools_acc": 0.9836,
   "needs_tools_auc": 0.991339380909933,
   "needs_vision_acc": 0.9982,
   "needs_vision_auc": 0.9998792270531401,
   "needs_long_context_acc": 0.9947,
   "needs_long_context_auc": 0.9995456251605128,
   "follow_up_acc": 0.9991,
   "follow_up_auc": 0.9997983309485168,
   "difficulty_spearman": 0.7386881928498473,
   "category_acc_by_source": {
    "arena55k": 0.8889,
    "synthetic-s1": 0.9125,
    "routellm-gpt4": 0.8503
   }
  },
  "B": {
   "n": 1134,
   "category_acc": 0.8854,
   "category_macro_f1": 0.8734,
   "needs_tools_acc": 0.9918,
   "needs_tools_auc": 0.9915403841784209,
   "needs_vision_acc": 0.9974,
   "needs_vision_auc": 0.9992149758454106,
   "needs_long_context_acc": 0.9938,
   "needs_long_context_auc": 0.9994468480214939,
   "follow_up_acc": 0.9991,
   "follow_up_auc": 0.9995006290153748,
   "difficulty_spearman": 0.7361477126530993,
   "category_acc_by_source": {
    "arena55k": 0.8679,
    "synthetic-s1": 0.9125,
    "routellm-gpt4": 0.8822
   }
  },
  "laya": {
   "n": 1134,
   "category_acc": 0.0935,
   "category_macro_f1": 0.112,
   "needs_tools_acc": 0.8046,
   "needs_tools_auc": 0.6708527782147414,
   "needs_vision_acc": 0.9594,
   "needs_vision_auc": 0.972403381642512,
   "needs_long_context_acc": 0.9466,
   "needs_long_context_auc": 0.8954345206345443,
   "follow_up_acc": 0.8834,
   "follow_up_auc": 0.8828494876645764,
   "difficulty_spearman": 0.2967021811737586,
   "category_acc_by_source": {
    "arena55k": 0.065,
    "synthetic-s1": 0.1924,
    "routellm-gpt4": 0.0287
   }
  },
  "jev": {
   "n": 1134,
   "category_acc": 0.858,
   "category_macro_f1": 0.8427,
   "needs_tools_acc": 0.9598,
   "needs_tools_auc": 0.9917763445370807,
   "needs_vision_acc": 0.9974,
   "needs_vision_auc": 1.0,
   "needs_long_context_acc": 0.9867,
   "needs_long_context_auc": 0.9971552183962543,
   "follow_up_acc": 0.9947,
   "follow_up_auc": 0.9985114903342904,
   "difficulty_spearman": 0.7532382322770758,
   "category_acc_by_source": {
    "arena55k": 0.8889,
    "synthetic-s1": 0.8717,
    "routellm-gpt4": 0.7962
   }
  }
 },
 "router70": {
  "A": {
   "n_cat": 59,
   "category_acc": 0.9492,
   "n": 70,
   "difficulty_llm_spearman": 0.190796955646925,
   "difficulty_outcome_spearman": 0.29776922403989153
  },
  "B": {
   "n_cat": 59,
   "category_acc": 0.9831,
   "n": 70,
   "difficulty_llm_spearman": 0.06482372495844632,
   "difficulty_outcome_spearman": 0.17910943924416062
  },
  "laya": {
   "n_cat": 59,
   "category_acc": 0.339,
   "n": 70,
   "difficulty_llm_spearman": -0.06321406701076021,
   "difficulty_outcome_spearman": -0.06321406701076021
  },
  "jev": {
   "n_cat": 59,
   "category_acc": 0.7966,
   "n": 70,
   "difficulty_llm_spearman": 0.29941387455165774,
   "difficulty_outcome_spearman": 0.29941387455165774
  }
 },
 "succ": {
  "routerbench": {
   "prior (heuristic)": {
    "small": {
     "auc": 0.0,
     "brier": 0.1666
    },
    "mid": {
     "auc": 0.0,
     "brier": 0.0567
    },
    "strong": {
     "auc": 0.0,
     "brier": 0.1017
    }
   },
   "A succ-heads": {
    "small": {
     "auc": 0.5871859132317802,
     "brier": 0.158
    },
    "mid": {
     "auc": 0.7284847213431849,
     "brier": 0.0455
    },
    "strong": {
     "auc": 0.6929741161200729,
     "brier": 0.085
    }
   },
   "A traits+calib": {
    "small": {
     "auc": 0.5390554995519713,
     "brier": 0.1623
    },
    "mid": {
     "auc": 0.6121454121361675,
     "brier": 0.0524
    },
    "strong": {
     "auc": 0.5743148952005791,
     "brier": 0.0964
    }
   },
   "B succ-heads": {
    "small": {
     "auc": 0.6710606145459976,
     "brier": 0.1491
    },
    "mid": {
     "auc": 0.7469368017579167,
     "brier": 0.0432
    },
    "strong": {
     "auc": 0.7181409533491113,
     "brier": 0.0816
    }
   },
   "B traits+calib": {
    "small": {
     "auc": 0.5797094347371565,
     "brier": 0.1586
    },
    "mid": {
     "auc": 0.6881226843785779,
     "brier": 0.0474
    },
    "strong": {
     "auc": 0.6694183558392462,
     "brier": 0.0898
    }
   },
   "laya traits+calib": {
    "small": {
     "auc": 0.512134109916368,
     "brier": 0.1646
    },
    "mid": {
     "auc": 0.5518876981389693,
     "brier": 0.0563
    },
    "strong": {
     "auc": 0.5477457402898577,
     "brier": 0.1021
    }
   },
   "jev traits+calib": {
    "small": {
     "auc": 0.5369670325567503,
     "brier": 0.1632
    },
    "mid": {
     "auc": 0.6381834861080493,
     "brier": 0.0528
    },
    "strong": {
     "auc": 0.5912175884220295,
     "brier": 0.0943
    }
   }
  },
  "measured-s2": {
   "prior (heuristic)": {
    "small": {
     "auc": 0.0,
     "brier": 0.2594
    },
    "mid": {
     "auc": 0.0,
     "brier": 0.2539
    },
    "strong": {
     "auc": 0.0,
     "brier": 0.1296
    }
   },
   "A succ-heads": {
    "small": {
     "auc": 0.9344919786096256,
     "brier": 0.0994
    },
    "mid": {
     "auc": 0.9594594594594594,
     "brier": 0.0348
    },
    "strong": {
     "auc": 0.9094567404426559,
     "brier": 0.0647
    }
   },
   "A traits+calib": {
    "small": {
     "auc": 0.9144385026737968,
     "brier": 0.1897
    },
    "mid": {
     "auc": 0.8412162162162162,
     "brier": 0.1478
    },
    "strong": {
     "auc": 0.7927565392354124,
     "brier": 0.0998
    }
   },
   "B succ-heads": {
    "small": {
     "auc": 0.9358288770053476,
     "brier": 0.0957
    },
    "mid": {
     "auc": 0.9425675675675675,
     "brier": 0.0308
    },
    "strong": {
     "auc": 0.8933601609657947,
     "brier": 0.0637
    }
   },
   "B traits+calib": {
    "small": {
     "auc": 0.9512032085561497,
     "brier": 0.1558
    },
    "mid": {
     "auc": 0.8141891891891891,
     "brier": 0.1351
    },
    "strong": {
     "auc": 0.8189134808853119,
     "brier": 0.0919
    }
   },
   "laya traits+calib": {
    "small": {
     "auc": 0.554144385026738,
     "brier": 0.2552
    },
    "mid": {
     "auc": 0.7567567567567568,
     "brier": 0.2061
    },
    "strong": {
     "auc": 0.2676056338028169,
     "brier": 0.1168
    }
   },
   "jev traits+calib": {
    "small": {
     "auc": 0.6724598930481284,
     "brier": 0.2268
    },
    "mid": {
     "auc": 0.7837837837837838,
     "brier": 0.1897
    },
    "strong": {
     "auc": 0.579476861167002,
     "brier": 0.1396
    }
   }
  }
 },
 "replay": {
  "routerbench": {
   "prior (heuristic)": {
    "n": 1500,
    "always_small": [
     4.6e-05,
     0.2845
    ],
    "always_mid": [
     0.0001341,
     0.4722
    ],
    "always_strong": [
     0.0031709,
     0.6862
    ],
    "oracle_quality": 0.7723,
    "best_quality": 0.6862,
    "quality_at_10pct_cost": 0.4722,
    "quality_at_25pct_cost": 0.4722,
    "quality_at_50pct_cost": 0.4722,
    "cost_share_for_80pct_gap": 1.0,
    "cost_share_for_95pct_gap": 1.0,
    "cost_share_for_100pct_gap": 1.0,
    "apgr": 0.4939
   },
   "oracle": {
    "n": 1500,
    "always_small": [
     4.6e-05,
     0.2845
    ],
    "always_mid": [
     0.0001341,
     0.4722
    ],
    "always_strong": [
     0.0031709,
     0.6862
    ],
    "oracle_quality": 0.7723,
    "best_quality": 0.7723,
    "quality_at_10pct_cost": 0.6457,
    "quality_at_25pct_cost": 0.7073,
    "quality_at_50pct_cost": 0.7666,
    "cost_share_for_80pct_gap": 0.0578,
    "cost_share_for_95pct_gap": 0.1468,
    "cost_share_for_100pct_gap": 0.1713,
    "apgr": 1.1403
   },
   "A succ-heads": {
    "n": 1500,
    "always_small": [
     4.6e-05,
     0.2845
    ],
    "always_mid": [
     0.0001341,
     0.4722
    ],
    "always_strong": [
     0.0031709,
     0.6862
    ],
    "oracle_quality": 0.7723,
    "best_quality": 0.6867,
    "quality_at_10pct_cost": 0.5137,
    "quality_at_25pct_cost": 0.5904,
    "quality_at_50pct_cost": 0.6601,
    "cost_share_for_80pct_gap": 0.3136,
    "cost_share_for_95pct_gap": 0.6521,
    "cost_share_for_100pct_gap": 0.9987,
    "apgr": 0.8618
   },
   "A traits+calib": {
    "n": 1500,
    "always_small": [
     4.6e-05,
     0.2845
    ],
    "always_mid": [
     0.0001341,
     0.4722
    ],
    "always_strong": [
     0.0031709,
     0.6862
    ],
    "oracle_quality": 0.7723,
    "best_quality": 0.6862,
    "quality_at_10pct_cost": 0.4873,
    "quality_at_25pct_cost": 0.582,
    "quality_at_50pct_cost": 0.6586,
    "cost_share_for_80pct_gap": 0.4096,
    "cost_share_for_95pct_gap": 0.8807,
    "cost_share_for_100pct_gap": 1.0,
    "apgr": 0.8335
   },
   "B succ-heads": {
    "n": 1500,
    "always_small": [
     4.6e-05,
     0.2845
    ],
    "always_mid": [
     0.0001341,
     0.4722
    ],
    "always_strong": [
     0.0031709,
     0.6862
    ],
    "oracle_quality": 0.7723,
    "best_quality": 0.6862,
    "quality_at_10pct_cost": 0.5179,
    "quality_at_25pct_cost": 0.5895,
    "quality_at_50pct_cost": 0.6512,
    "cost_share_for_80pct_gap": 0.3039,
    "cost_share_for_95pct_gap": 0.6865,
    "cost_share_for_100pct_gap": 1.0,
    "apgr": 0.8638
   },
   "B traits+calib": {
    "n": 1500,
    "always_small": [
     4.6e-05,
     0.2845
    ],
    "always_mid": [
     0.0001341,
     0.4722
    ],
    "always_strong": [
     0.0031709,
     0.6862
    ],
    "oracle_quality": 0.7723,
    "best_quality": 0.6862,
    "quality_at_10pct_cost": 0.4978,
    "quality_at_25pct_cost": 0.5818,
    "quality_at_50pct_cost": 0.6593,
    "cost_share_for_80pct_gap": 0.3795,
    "cost_share_for_95pct_gap": 0.8564,
    "cost_share_for_100pct_gap": 1.0,
    "apgr": 0.8411
   },
   "laya traits+calib": {
    "n": 1500,
    "always_small": [
     4.6e-05,
     0.2845
    ],
    "always_mid": [
     0.0001341,
     0.4722
    ],
    "always_strong": [
     0.0031709,
     0.6862
    ],
    "oracle_quality": 0.7723,
    "best_quality": 0.6862,
    "quality_at_10pct_cost": 0.4892,
    "quality_at_25pct_cost": 0.5384,
    "quality_at_50pct_cost": 0.6349,
    "cost_share_for_80pct_gap": 0.3893,
    "cost_share_for_95pct_gap": 0.7946,
    "cost_share_for_100pct_gap": 1.0,
    "apgr": 0.8125
   },
   "jev traits+calib": {
    "n": 1500,
    "always_small": [
     4.6e-05,
     0.2845
    ],
    "always_mid": [
     0.0001341,
     0.4722
    ],
    "always_strong": [
     0.0031709,
     0.6862
    ],
    "oracle_quality": 0.7723,
    "best_quality": 0.6862,
    "quality_at_10pct_cost": 0.503,
    "quality_at_25pct_cost": 0.596,
    "quality_at_50pct_cost": 0.6555,
    "cost_share_for_80pct_gap": 0.2817,
    "cost_share_for_95pct_gap": 0.8623,
    "cost_share_for_100pct_gap": 1.0,
    "apgr": 0.8499
   }
  },
  "measured-s2": {
   "prior (heuristic)": {
    "n": 78,
    "always_small": [
     1.14e-05,
     0.4359
    ],
    "always_mid": [
     0.0001846,
     0.9487
    ],
    "always_strong": [
     0.0003176,
     0.9103
    ],
    "oracle_quality": 0.9872,
    "best_quality": 0.9487,
    "quality_at_10pct_cost": 0.4359,
    "quality_at_25pct_cost": 0.4359,
    "quality_at_50pct_cost": 0.4359,
    "cost_share_for_80pct_gap": 0.5814,
    "cost_share_for_95pct_gap": 0.5814,
    "cost_share_for_100pct_gap": 0.5814,
    "apgr": 0.4865
   },
   "oracle": {
    "n": 78,
    "always_small": [
     1.14e-05,
     0.4359
    ],
    "always_mid": [
     0.0001846,
     0.9487
    ],
    "always_strong": [
     0.0003176,
     0.9103
    ],
    "oracle_quality": 0.9872,
    "best_quality": 0.9872,
    "quality_at_10pct_cost": 0.4359,
    "quality_at_25pct_cost": 0.4359,
    "quality_at_50pct_cost": 0.9872,
    "cost_share_for_80pct_gap": 0.3877,
    "cost_share_for_95pct_gap": 0.3877,
    "cost_share_for_100pct_gap": 0.3877,
    "apgr": 0.7514
   },
   "A succ-heads": {
    "n": 78,
    "always_small": [
     1.14e-05,
     0.4359
    ],
    "always_mid": [
     0.0001846,
     0.9487
    ],
    "always_strong": [
     0.0003176,
     0.9103
    ],
    "oracle_quality": 0.9872,
    "best_quality": 0.9872,
    "quality_at_10pct_cost": 0.5385,
    "quality_at_25pct_cost": 0.7179,
    "quality_at_50pct_cost": 0.9231,
    "cost_share_for_80pct_gap": 0.3784,
    "cost_share_for_95pct_gap": 0.4781,
    "cost_share_for_100pct_gap": 0.4913,
    "apgr": 0.8986
   },
   "A traits+calib": {
    "n": 78,
    "always_small": [
     1.14e-05,
     0.4359
    ],
    "always_mid": [
     0.0001846,
     0.9487
    ],
    "always_strong": [
     0.0003176,
     0.9103
    ],
    "oracle_quality": 0.9872,
    "best_quality": 0.9487,
    "quality_at_10pct_cost": 0.5128,
    "quality_at_25pct_cost": 0.5128,
    "quality_at_50pct_cost": 0.6667,
    "cost_share_for_80pct_gap": 0.5814,
    "cost_share_for_95pct_gap": 0.5814,
    "cost_share_for_100pct_gap": 0.5814,
    "apgr": 0.6608
   },
   "B succ-heads": {
    "n": 78,
    "always_small": [
     1.14e-05,
     0.4359
    ],
    "always_mid": [
     0.0001846,
     0.9487
    ],
    "always_strong": [
     0.0003176,
     0.9103
    ],
    "oracle_quality": 0.9872,
    "best_quality": 0.9872,
    "quality_at_10pct_cost": 0.5513,
    "quality_at_25pct_cost": 0.6795,
    "quality_at_50pct_cost": 0.9103,
    "cost_share_for_80pct_gap": 0.3551,
    "cost_share_for_95pct_gap": 0.4647,
    "cost_share_for_100pct_gap": 0.495,
    "apgr": 0.8959
   },
   "B traits+calib": {
    "n": 78,
    "always_small": [
     1.14e-05,
     0.4359
    ],
    "always_mid": [
     0.0001846,
     0.9487
    ],
    "always_strong": [
     0.0003176,
     0.9103
    ],
    "oracle_quality": 0.9872,
    "best_quality": 0.9487,
    "quality_at_10pct_cost": 0.5256,
    "quality_at_25pct_cost": 0.5769,
    "quality_at_50pct_cost": 0.6795,
    "cost_share_for_80pct_gap": 0.5814,
    "cost_share_for_95pct_gap": 0.5814,
    "cost_share_for_100pct_gap": 0.5814,
    "apgr": 0.6797
   },
   "laya traits+calib": {
    "n": 78,
    "always_small": [
     1.14e-05,
     0.4359
    ],
    "always_mid": [
     0.0001846,
     0.9487
    ],
    "always_strong": [
     0.0003176,
     0.9103
    ],
    "oracle_quality": 0.9872,
    "best_quality": 0.9487,
    "quality_at_10pct_cost": 0.4615,
    "quality_at_25pct_cost": 0.5385,
    "quality_at_50pct_cost": 0.6282,
    "cost_share_for_80pct_gap": 0.5814,
    "cost_share_for_95pct_gap": 0.5814,
    "cost_share_for_100pct_gap": 0.5814,
    "apgr": 0.6446
   },
   "jev traits+calib": {
    "n": 78,
    "always_small": [
     1.14e-05,
     0.4359
    ],
    "always_mid": [
     0.0001846,
     0.9487
    ],
    "always_strong": [
     0.0003176,
     0.9103
    ],
    "oracle_quality": 0.9872,
    "best_quality": 0.9487,
    "quality_at_10pct_cost": 0.4487,
    "quality_at_25pct_cost": 0.5256,
    "quality_at_50pct_cost": 0.8077,
    "cost_share_for_80pct_gap": 0.5113,
    "cost_share_for_95pct_gap": 0.5814,
    "cost_share_for_100pct_gap": 0.5814,
    "apgr": 0.7311
   }
  }
 },
 "route": {
  "A": {
   "route_head": {
    "n": 6586,
    "oracle_agreement": 0.6731,
    "mean_regret_usd": 0.0502961
   },
   "econ[prior (heuristic)]": {
    "n": 6586,
    "oracle_agreement": 0.741,
    "mean_regret_usd": 0.0350319
   },
   "econ[oracle]": {
    "n": 6586,
    "oracle_agreement": 0.9368,
    "mean_regret_usd": 0.0216345
   },
   "econ[A succ-heads]": {
    "n": 6586,
    "oracle_agreement": 0.7475,
    "mean_regret_usd": 0.0348688
   },
   "econ[A traits+calib]": {
    "n": 6586,
    "oracle_agreement": 0.7426,
    "mean_regret_usd": 0.0347834
   },
   "econ[laya traits+calib]": {
    "n": 3156,
    "oracle_agreement": 0.7335,
    "mean_regret_usd": 0.0371752
   },
   "econ[jev traits+calib]": {
    "n": 3156,
    "oracle_agreement": 0.7411,
    "mean_regret_usd": 0.0369512
   },
   "always_strong": {
    "n": 6586,
    "oracle_agreement": 0.136,
    "mean_regret_usd": 0.1993001
   },
   "always_small": {
    "n": 6586,
    "oracle_agreement": 0.5742,
    "mean_regret_usd": 0.069763
   }
  },
  "B": {
   "route_head": {
    "n": 6586,
    "oracle_agreement": 0.6828,
    "mean_regret_usd": 0.0488386
   },
   "econ[prior (heuristic)]": {
    "n": 6586,
    "oracle_agreement": 0.741,
    "mean_regret_usd": 0.0350319
   },
   "econ[oracle]": {
    "n": 6586,
    "oracle_agreement": 0.9368,
    "mean_regret_usd": 0.0216345
   },
   "econ[B succ-heads]": {
    "n": 6586,
    "oracle_agreement": 0.7589,
    "mean_regret_usd": 0.0346441
   },
   "econ[B traits+calib]": {
    "n": 6586,
    "oracle_agreement": 0.7431,
    "mean_regret_usd": 0.0347052
   },
   "econ[laya traits+calib]": {
    "n": 3156,
    "oracle_agreement": 0.7335,
    "mean_regret_usd": 0.0371752
   },
   "econ[jev traits+calib]": {
    "n": 3156,
    "oracle_agreement": 0.7411,
    "mean_regret_usd": 0.0369512
   },
   "always_strong": {
    "n": 6586,
    "oracle_agreement": 0.136,
    "mean_regret_usd": 0.1993001
   },
   "always_small": {
    "n": 6586,
    "oracle_agreement": 0.5742,
    "mean_regret_usd": 0.069763
   }
  }
 },
 "latency_ms_batched_gpu": {
  "A": 6.219,
  "B": 10.11,
  "laya": 24.631,
  "jev": 274.5
 }
}