| { |
| "n_test": 13632, |
| "n_calib": 5572, |
| "traits": { |
| "A": { |
| "n": 1134, |
| "category_acc": 0.8854, |
| "category_macro_f1": 0.8722, |
| "needs_tools_acc": 0.9836, |
| "needs_tools_auc": 0.991339380909933, |
| "needs_vision_acc": 0.9982, |
| "needs_vision_auc": 0.9998792270531401, |
| "needs_long_context_acc": 0.9947, |
| "needs_long_context_auc": 0.9995456251605128, |
| "follow_up_acc": 0.9991, |
| "follow_up_auc": 0.9997983309485168, |
| "difficulty_spearman": 0.7386881928498473, |
| "category_acc_by_source": { |
| "arena55k": 0.8889, |
| "synthetic-s1": 0.9125, |
| "routellm-gpt4": 0.8503 |
| } |
| }, |
| "B": { |
| "n": 1134, |
| "category_acc": 0.8854, |
| "category_macro_f1": 0.8734, |
| "needs_tools_acc": 0.9918, |
| "needs_tools_auc": 0.9915403841784209, |
| "needs_vision_acc": 0.9974, |
| "needs_vision_auc": 0.9992149758454106, |
| "needs_long_context_acc": 0.9938, |
| "needs_long_context_auc": 0.9994468480214939, |
| "follow_up_acc": 0.9991, |
| "follow_up_auc": 0.9995006290153748, |
| "difficulty_spearman": 0.7361477126530993, |
| "category_acc_by_source": { |
| "arena55k": 0.8679, |
| "synthetic-s1": 0.9125, |
| "routellm-gpt4": 0.8822 |
| } |
| }, |
| "laya": { |
| "n": 1134, |
| "category_acc": 0.0935, |
| "category_macro_f1": 0.112, |
| "needs_tools_acc": 0.8046, |
| "needs_tools_auc": 0.6708527782147414, |
| "needs_vision_acc": 0.9594, |
| "needs_vision_auc": 0.972403381642512, |
| "needs_long_context_acc": 0.9466, |
| "needs_long_context_auc": 0.8954345206345443, |
| "follow_up_acc": 0.8834, |
| "follow_up_auc": 0.8828494876645764, |
| "difficulty_spearman": 0.2967021811737586, |
| "category_acc_by_source": { |
| "arena55k": 0.065, |
| "synthetic-s1": 0.1924, |
| "routellm-gpt4": 0.0287 |
| } |
| }, |
| "jev": { |
| "n": 1134, |
| "category_acc": 0.858, |
| "category_macro_f1": 0.8427, |
| "needs_tools_acc": 0.9598, |
| "needs_tools_auc": 0.9917763445370807, |
| "needs_vision_acc": 0.9974, |
| "needs_vision_auc": 1.0, |
| "needs_long_context_acc": 0.9867, |
| "needs_long_context_auc": 0.9971552183962543, |
| "follow_up_acc": 0.9947, |
| "follow_up_auc": 0.9985114903342904, |
| "difficulty_spearman": 0.7532382322770758, |
| "category_acc_by_source": { |
| "arena55k": 0.8889, |
| "synthetic-s1": 0.8717, |
| "routellm-gpt4": 0.7962 |
| } |
| } |
| }, |
| "router70": { |
| "A": { |
| "n_cat": 59, |
| "category_acc": 0.9492, |
| "n": 70, |
| "difficulty_llm_spearman": 0.190796955646925, |
| "difficulty_outcome_spearman": 0.29776922403989153 |
| }, |
| "B": { |
| "n_cat": 59, |
| "category_acc": 0.9831, |
| "n": 70, |
| "difficulty_llm_spearman": 0.06482372495844632, |
| "difficulty_outcome_spearman": 0.17910943924416062 |
| }, |
| "laya": { |
| "n_cat": 59, |
| "category_acc": 0.339, |
| "n": 70, |
| "difficulty_llm_spearman": -0.06321406701076021, |
| "difficulty_outcome_spearman": -0.06321406701076021 |
| }, |
| "jev": { |
| "n_cat": 59, |
| "category_acc": 0.7966, |
| "n": 70, |
| "difficulty_llm_spearman": 0.29941387455165774, |
| "difficulty_outcome_spearman": 0.29941387455165774 |
| } |
| }, |
| "succ": { |
| "routerbench": { |
| "prior (heuristic)": { |
| "small": { |
| "auc": 0.0, |
| "brier": 0.1666 |
| }, |
| "mid": { |
| "auc": 0.0, |
| "brier": 0.0567 |
| }, |
| "strong": { |
| "auc": 0.0, |
| "brier": 0.1017 |
| } |
| }, |
| "A succ-heads": { |
| "small": { |
| "auc": 0.5871859132317802, |
| "brier": 0.158 |
| }, |
| "mid": { |
| "auc": 0.7284847213431849, |
| "brier": 0.0455 |
| }, |
| "strong": { |
| "auc": 0.6929741161200729, |
| "brier": 0.085 |
| } |
| }, |
| "A traits+calib": { |
| "small": { |
| "auc": 0.5390554995519713, |
| "brier": 0.1623 |
| }, |
| "mid": { |
| "auc": 0.6121454121361675, |
| "brier": 0.0524 |
| }, |
| "strong": { |
| "auc": 0.5743148952005791, |
| "brier": 0.0964 |
| } |
| }, |
| "B succ-heads": { |
| "small": { |
| "auc": 0.6710606145459976, |
| "brier": 0.1491 |
| }, |
| "mid": { |
| "auc": 0.7469368017579167, |
| "brier": 0.0432 |
| }, |
| "strong": { |
| "auc": 0.7181409533491113, |
| "brier": 0.0816 |
| } |
| }, |
| "B traits+calib": { |
| "small": { |
| "auc": 0.5797094347371565, |
| "brier": 0.1586 |
| }, |
| "mid": { |
| "auc": 0.6881226843785779, |
| "brier": 0.0474 |
| }, |
| "strong": { |
| "auc": 0.6694183558392462, |
| "brier": 0.0898 |
| } |
| }, |
| "laya traits+calib": { |
| "small": { |
| "auc": 0.512134109916368, |
| "brier": 0.1646 |
| }, |
| "mid": { |
| "auc": 0.5518876981389693, |
| "brier": 0.0563 |
| }, |
| "strong": { |
| "auc": 0.5477457402898577, |
| "brier": 0.1021 |
| } |
| }, |
| "jev traits+calib": { |
| "small": { |
| "auc": 0.5369670325567503, |
| "brier": 0.1632 |
| }, |
| "mid": { |
| "auc": 0.6381834861080493, |
| "brier": 0.0528 |
| }, |
| "strong": { |
| "auc": 0.5912175884220295, |
| "brier": 0.0943 |
| } |
| } |
| }, |
| "measured-s2": { |
| "prior (heuristic)": { |
| "small": { |
| "auc": 0.0, |
| "brier": 0.2594 |
| }, |
| "mid": { |
| "auc": 0.0, |
| "brier": 0.2539 |
| }, |
| "strong": { |
| "auc": 0.0, |
| "brier": 0.1296 |
| } |
| }, |
| "A succ-heads": { |
| "small": { |
| "auc": 0.9344919786096256, |
| "brier": 0.0994 |
| }, |
| "mid": { |
| "auc": 0.9594594594594594, |
| "brier": 0.0348 |
| }, |
| "strong": { |
| "auc": 0.9094567404426559, |
| "brier": 0.0647 |
| } |
| }, |
| "A traits+calib": { |
| "small": { |
| "auc": 0.9144385026737968, |
| "brier": 0.1897 |
| }, |
| "mid": { |
| "auc": 0.8412162162162162, |
| "brier": 0.1478 |
| }, |
| "strong": { |
| "auc": 0.7927565392354124, |
| "brier": 0.0998 |
| } |
| }, |
| "B succ-heads": { |
| "small": { |
| "auc": 0.9358288770053476, |
| "brier": 0.0957 |
| }, |
| "mid": { |
| "auc": 0.9425675675675675, |
| "brier": 0.0308 |
| }, |
| "strong": { |
| "auc": 0.8933601609657947, |
| "brier": 0.0637 |
| } |
| }, |
| "B traits+calib": { |
| "small": { |
| "auc": 0.9512032085561497, |
| "brier": 0.1558 |
| }, |
| "mid": { |
| "auc": 0.8141891891891891, |
| "brier": 0.1351 |
| }, |
| "strong": { |
| "auc": 0.8189134808853119, |
| "brier": 0.0919 |
| } |
| }, |
| "laya traits+calib": { |
| "small": { |
| "auc": 0.554144385026738, |
| "brier": 0.2552 |
| }, |
| "mid": { |
| "auc": 0.7567567567567568, |
| "brier": 0.2061 |
| }, |
| "strong": { |
| "auc": 0.2676056338028169, |
| "brier": 0.1168 |
| } |
| }, |
| "jev traits+calib": { |
| "small": { |
| "auc": 0.6724598930481284, |
| "brier": 0.2268 |
| }, |
| "mid": { |
| "auc": 0.7837837837837838, |
| "brier": 0.1897 |
| }, |
| "strong": { |
| "auc": 0.579476861167002, |
| "brier": 0.1396 |
| } |
| } |
| } |
| }, |
| "replay": { |
| "routerbench": { |
| "prior (heuristic)": { |
| "n": 1500, |
| "always_small": [ |
| 4.6e-05, |
| 0.2845 |
| ], |
| "always_mid": [ |
| 0.0001341, |
| 0.4722 |
| ], |
| "always_strong": [ |
| 0.0031709, |
| 0.6862 |
| ], |
| "oracle_quality": 0.7723, |
| "best_quality": 0.6862, |
| "quality_at_10pct_cost": 0.4722, |
| "quality_at_25pct_cost": 0.4722, |
| "quality_at_50pct_cost": 0.4722, |
| "cost_share_for_80pct_gap": 1.0, |
| "cost_share_for_95pct_gap": 1.0, |
| "cost_share_for_100pct_gap": 1.0, |
| "apgr": 0.4939 |
| }, |
| "oracle": { |
| "n": 1500, |
| "always_small": [ |
| 4.6e-05, |
| 0.2845 |
| ], |
| "always_mid": [ |
| 0.0001341, |
| 0.4722 |
| ], |
| "always_strong": [ |
| 0.0031709, |
| 0.6862 |
| ], |
| "oracle_quality": 0.7723, |
| "best_quality": 0.7723, |
| "quality_at_10pct_cost": 0.6457, |
| "quality_at_25pct_cost": 0.7073, |
| "quality_at_50pct_cost": 0.7666, |
| "cost_share_for_80pct_gap": 0.0578, |
| "cost_share_for_95pct_gap": 0.1468, |
| "cost_share_for_100pct_gap": 0.1713, |
| "apgr": 1.1403 |
| }, |
| "A succ-heads": { |
| "n": 1500, |
| "always_small": [ |
| 4.6e-05, |
| 0.2845 |
| ], |
| "always_mid": [ |
| 0.0001341, |
| 0.4722 |
| ], |
| "always_strong": [ |
| 0.0031709, |
| 0.6862 |
| ], |
| "oracle_quality": 0.7723, |
| "best_quality": 0.6867, |
| "quality_at_10pct_cost": 0.5137, |
| "quality_at_25pct_cost": 0.5904, |
| "quality_at_50pct_cost": 0.6601, |
| "cost_share_for_80pct_gap": 0.3136, |
| "cost_share_for_95pct_gap": 0.6521, |
| "cost_share_for_100pct_gap": 0.9987, |
| "apgr": 0.8618 |
| }, |
| "A traits+calib": { |
| "n": 1500, |
| "always_small": [ |
| 4.6e-05, |
| 0.2845 |
| ], |
| "always_mid": [ |
| 0.0001341, |
| 0.4722 |
| ], |
| "always_strong": [ |
| 0.0031709, |
| 0.6862 |
| ], |
| "oracle_quality": 0.7723, |
| "best_quality": 0.6862, |
| "quality_at_10pct_cost": 0.4873, |
| "quality_at_25pct_cost": 0.582, |
| "quality_at_50pct_cost": 0.6586, |
| "cost_share_for_80pct_gap": 0.4096, |
| "cost_share_for_95pct_gap": 0.8807, |
| "cost_share_for_100pct_gap": 1.0, |
| "apgr": 0.8335 |
| }, |
| "B succ-heads": { |
| "n": 1500, |
| "always_small": [ |
| 4.6e-05, |
| 0.2845 |
| ], |
| "always_mid": [ |
| 0.0001341, |
| 0.4722 |
| ], |
| "always_strong": [ |
| 0.0031709, |
| 0.6862 |
| ], |
| "oracle_quality": 0.7723, |
| "best_quality": 0.6862, |
| "quality_at_10pct_cost": 0.5179, |
| "quality_at_25pct_cost": 0.5895, |
| "quality_at_50pct_cost": 0.6512, |
| "cost_share_for_80pct_gap": 0.3039, |
| "cost_share_for_95pct_gap": 0.6865, |
| "cost_share_for_100pct_gap": 1.0, |
| "apgr": 0.8638 |
| }, |
| "B traits+calib": { |
| "n": 1500, |
| "always_small": [ |
| 4.6e-05, |
| 0.2845 |
| ], |
| "always_mid": [ |
| 0.0001341, |
| 0.4722 |
| ], |
| "always_strong": [ |
| 0.0031709, |
| 0.6862 |
| ], |
| "oracle_quality": 0.7723, |
| "best_quality": 0.6862, |
| "quality_at_10pct_cost": 0.4978, |
| "quality_at_25pct_cost": 0.5818, |
| "quality_at_50pct_cost": 0.6593, |
| "cost_share_for_80pct_gap": 0.3795, |
| "cost_share_for_95pct_gap": 0.8564, |
| "cost_share_for_100pct_gap": 1.0, |
| "apgr": 0.8411 |
| }, |
| "laya traits+calib": { |
| "n": 1500, |
| "always_small": [ |
| 4.6e-05, |
| 0.2845 |
| ], |
| "always_mid": [ |
| 0.0001341, |
| 0.4722 |
| ], |
| "always_strong": [ |
| 0.0031709, |
| 0.6862 |
| ], |
| "oracle_quality": 0.7723, |
| "best_quality": 0.6862, |
| "quality_at_10pct_cost": 0.4892, |
| "quality_at_25pct_cost": 0.5384, |
| "quality_at_50pct_cost": 0.6349, |
| "cost_share_for_80pct_gap": 0.3893, |
| "cost_share_for_95pct_gap": 0.7946, |
| "cost_share_for_100pct_gap": 1.0, |
| "apgr": 0.8125 |
| }, |
| "jev traits+calib": { |
| "n": 1500, |
| "always_small": [ |
| 4.6e-05, |
| 0.2845 |
| ], |
| "always_mid": [ |
| 0.0001341, |
| 0.4722 |
| ], |
| "always_strong": [ |
| 0.0031709, |
| 0.6862 |
| ], |
| "oracle_quality": 0.7723, |
| "best_quality": 0.6862, |
| "quality_at_10pct_cost": 0.503, |
| "quality_at_25pct_cost": 0.596, |
| "quality_at_50pct_cost": 0.6555, |
| "cost_share_for_80pct_gap": 0.2817, |
| "cost_share_for_95pct_gap": 0.8623, |
| "cost_share_for_100pct_gap": 1.0, |
| "apgr": 0.8499 |
| } |
| }, |
| "measured-s2": { |
| "prior (heuristic)": { |
| "n": 78, |
| "always_small": [ |
| 1.14e-05, |
| 0.4359 |
| ], |
| "always_mid": [ |
| 0.0001846, |
| 0.9487 |
| ], |
| "always_strong": [ |
| 0.0003176, |
| 0.9103 |
| ], |
| "oracle_quality": 0.9872, |
| "best_quality": 0.9487, |
| "quality_at_10pct_cost": 0.4359, |
| "quality_at_25pct_cost": 0.4359, |
| "quality_at_50pct_cost": 0.4359, |
| "cost_share_for_80pct_gap": 0.5814, |
| "cost_share_for_95pct_gap": 0.5814, |
| "cost_share_for_100pct_gap": 0.5814, |
| "apgr": 0.4865 |
| }, |
| "oracle": { |
| "n": 78, |
| "always_small": [ |
| 1.14e-05, |
| 0.4359 |
| ], |
| "always_mid": [ |
| 0.0001846, |
| 0.9487 |
| ], |
| "always_strong": [ |
| 0.0003176, |
| 0.9103 |
| ], |
| "oracle_quality": 0.9872, |
| "best_quality": 0.9872, |
| "quality_at_10pct_cost": 0.4359, |
| "quality_at_25pct_cost": 0.4359, |
| "quality_at_50pct_cost": 0.9872, |
| "cost_share_for_80pct_gap": 0.3877, |
| "cost_share_for_95pct_gap": 0.3877, |
| "cost_share_for_100pct_gap": 0.3877, |
| "apgr": 0.7514 |
| }, |
| "A succ-heads": { |
| "n": 78, |
| "always_small": [ |
| 1.14e-05, |
| 0.4359 |
| ], |
| "always_mid": [ |
| 0.0001846, |
| 0.9487 |
| ], |
| "always_strong": [ |
| 0.0003176, |
| 0.9103 |
| ], |
| "oracle_quality": 0.9872, |
| "best_quality": 0.9872, |
| "quality_at_10pct_cost": 0.5385, |
| "quality_at_25pct_cost": 0.7179, |
| "quality_at_50pct_cost": 0.9231, |
| "cost_share_for_80pct_gap": 0.3784, |
| "cost_share_for_95pct_gap": 0.4781, |
| "cost_share_for_100pct_gap": 0.4913, |
| "apgr": 0.8986 |
| }, |
| "A traits+calib": { |
| "n": 78, |
| "always_small": [ |
| 1.14e-05, |
| 0.4359 |
| ], |
| "always_mid": [ |
| 0.0001846, |
| 0.9487 |
| ], |
| "always_strong": [ |
| 0.0003176, |
| 0.9103 |
| ], |
| "oracle_quality": 0.9872, |
| "best_quality": 0.9487, |
| "quality_at_10pct_cost": 0.5128, |
| "quality_at_25pct_cost": 0.5128, |
| "quality_at_50pct_cost": 0.6667, |
| "cost_share_for_80pct_gap": 0.5814, |
| "cost_share_for_95pct_gap": 0.5814, |
| "cost_share_for_100pct_gap": 0.5814, |
| "apgr": 0.6608 |
| }, |
| "B succ-heads": { |
| "n": 78, |
| "always_small": [ |
| 1.14e-05, |
| 0.4359 |
| ], |
| "always_mid": [ |
| 0.0001846, |
| 0.9487 |
| ], |
| "always_strong": [ |
| 0.0003176, |
| 0.9103 |
| ], |
| "oracle_quality": 0.9872, |
| "best_quality": 0.9872, |
| "quality_at_10pct_cost": 0.5513, |
| "quality_at_25pct_cost": 0.6795, |
| "quality_at_50pct_cost": 0.9103, |
| "cost_share_for_80pct_gap": 0.3551, |
| "cost_share_for_95pct_gap": 0.4647, |
| "cost_share_for_100pct_gap": 0.495, |
| "apgr": 0.8959 |
| }, |
| "B traits+calib": { |
| "n": 78, |
| "always_small": [ |
| 1.14e-05, |
| 0.4359 |
| ], |
| "always_mid": [ |
| 0.0001846, |
| 0.9487 |
| ], |
| "always_strong": [ |
| 0.0003176, |
| 0.9103 |
| ], |
| "oracle_quality": 0.9872, |
| "best_quality": 0.9487, |
| "quality_at_10pct_cost": 0.5256, |
| "quality_at_25pct_cost": 0.5769, |
| "quality_at_50pct_cost": 0.6795, |
| "cost_share_for_80pct_gap": 0.5814, |
| "cost_share_for_95pct_gap": 0.5814, |
| "cost_share_for_100pct_gap": 0.5814, |
| "apgr": 0.6797 |
| }, |
| "laya traits+calib": { |
| "n": 78, |
| "always_small": [ |
| 1.14e-05, |
| 0.4359 |
| ], |
| "always_mid": [ |
| 0.0001846, |
| 0.9487 |
| ], |
| "always_strong": [ |
| 0.0003176, |
| 0.9103 |
| ], |
| "oracle_quality": 0.9872, |
| "best_quality": 0.9487, |
| "quality_at_10pct_cost": 0.4615, |
| "quality_at_25pct_cost": 0.5385, |
| "quality_at_50pct_cost": 0.6282, |
| "cost_share_for_80pct_gap": 0.5814, |
| "cost_share_for_95pct_gap": 0.5814, |
| "cost_share_for_100pct_gap": 0.5814, |
| "apgr": 0.6446 |
| }, |
| "jev traits+calib": { |
| "n": 78, |
| "always_small": [ |
| 1.14e-05, |
| 0.4359 |
| ], |
| "always_mid": [ |
| 0.0001846, |
| 0.9487 |
| ], |
| "always_strong": [ |
| 0.0003176, |
| 0.9103 |
| ], |
| "oracle_quality": 0.9872, |
| "best_quality": 0.9487, |
| "quality_at_10pct_cost": 0.4487, |
| "quality_at_25pct_cost": 0.5256, |
| "quality_at_50pct_cost": 0.8077, |
| "cost_share_for_80pct_gap": 0.5113, |
| "cost_share_for_95pct_gap": 0.5814, |
| "cost_share_for_100pct_gap": 0.5814, |
| "apgr": 0.7311 |
| } |
| } |
| }, |
| "route": { |
| "A": { |
| "route_head": { |
| "n": 6586, |
| "oracle_agreement": 0.6731, |
| "mean_regret_usd": 0.0502961 |
| }, |
| "econ[prior (heuristic)]": { |
| "n": 6586, |
| "oracle_agreement": 0.741, |
| "mean_regret_usd": 0.0350319 |
| }, |
| "econ[oracle]": { |
| "n": 6586, |
| "oracle_agreement": 0.9368, |
| "mean_regret_usd": 0.0216345 |
| }, |
| "econ[A succ-heads]": { |
| "n": 6586, |
| "oracle_agreement": 0.7475, |
| "mean_regret_usd": 0.0348688 |
| }, |
| "econ[A traits+calib]": { |
| "n": 6586, |
| "oracle_agreement": 0.7426, |
| "mean_regret_usd": 0.0347834 |
| }, |
| "econ[laya traits+calib]": { |
| "n": 3156, |
| "oracle_agreement": 0.7335, |
| "mean_regret_usd": 0.0371752 |
| }, |
| "econ[jev traits+calib]": { |
| "n": 3156, |
| "oracle_agreement": 0.7411, |
| "mean_regret_usd": 0.0369512 |
| }, |
| "always_strong": { |
| "n": 6586, |
| "oracle_agreement": 0.136, |
| "mean_regret_usd": 0.1993001 |
| }, |
| "always_small": { |
| "n": 6586, |
| "oracle_agreement": 0.5742, |
| "mean_regret_usd": 0.069763 |
| } |
| }, |
| "B": { |
| "route_head": { |
| "n": 6586, |
| "oracle_agreement": 0.6828, |
| "mean_regret_usd": 0.0488386 |
| }, |
| "econ[prior (heuristic)]": { |
| "n": 6586, |
| "oracle_agreement": 0.741, |
| "mean_regret_usd": 0.0350319 |
| }, |
| "econ[oracle]": { |
| "n": 6586, |
| "oracle_agreement": 0.9368, |
| "mean_regret_usd": 0.0216345 |
| }, |
| "econ[B succ-heads]": { |
| "n": 6586, |
| "oracle_agreement": 0.7589, |
| "mean_regret_usd": 0.0346441 |
| }, |
| "econ[B traits+calib]": { |
| "n": 6586, |
| "oracle_agreement": 0.7431, |
| "mean_regret_usd": 0.0347052 |
| }, |
| "econ[laya traits+calib]": { |
| "n": 3156, |
| "oracle_agreement": 0.7335, |
| "mean_regret_usd": 0.0371752 |
| }, |
| "econ[jev traits+calib]": { |
| "n": 3156, |
| "oracle_agreement": 0.7411, |
| "mean_regret_usd": 0.0369512 |
| }, |
| "always_strong": { |
| "n": 6586, |
| "oracle_agreement": 0.136, |
| "mean_regret_usd": 0.1993001 |
| }, |
| "always_small": { |
| "n": 6586, |
| "oracle_agreement": 0.5742, |
| "mean_regret_usd": 0.069763 |
| } |
| } |
| }, |
| "latency_ms_batched_gpu": { |
| "A": 6.219, |
| "B": 10.11, |
| "laya": 24.631, |
| "jev": 274.5 |
| } |
| } |
