{ "post_T": 1.0399999618530273, "quality": { "train": { "ag_news": { "acc": 0.926, "nll": 0.23406508564949036, "ece": 0.0323681704998016, "brier": 0.11545059829950333, "n": 500 }, "banking77": { "acc": 0.902, "nll": 0.3612114489078522, "ece": 0.035957046896219286, "brier": 0.15479955077171326, "n": 500 }, "emotion": { "acc": 0.922, "nll": 0.15576711297035217, "ece": 0.030244563996791818, "brier": 0.08809012919664383, "n": 500 }, "imdb": { "acc": 0.886, "nll": 0.2880212068557739, "ece": 0.027435769557952863, "brier": 0.1726732850074768, "n": 500 }, "mrpc": { "acc": 0.8217054263565892, "nll": 0.3895823657512665, "ece": 0.042235060486682595, "brier": 0.25083833932876587, "n": 258 }, "news20": { "acc": 0.674, "nll": 1.0884853601455688, "ece": 0.06790356469154359, "brier": 0.4469330608844757, "n": 500 }, "sms": { "acc": 0.9926470588235294, "nll": 0.038313742727041245, "ece": 0.0037884642096127656, "brier": 0.015446269884705544, "n": 408 }, "snli": { "acc": 0.772, "nll": 0.5716506838798523, "ece": 0.045889752805233, "brier": 0.3297711908817291, "n": 500 }, "sst2": { "acc": 0.912, "nll": 0.24116064608097076, "ece": 0.034141219377517684, "brier": 0.14030593633651733, "n": 500 }, "yelp": { "acc": 0.572, "nll": 0.9883294105529785, "ece": 0.0719212611913681, "brier": 0.5307314991950989, "n": 500 }, "_macro": { "acc": 0.8380352485180118, "nll": 0.4356587529182434, "ece": 0.01943733276175781, "brier": 0.2245039939880371 } }, "unseen": { "ag_news": { "acc": 0.924, "nll": 0.25852325558662415, "ece": 0.041514035522937774, "brier": 0.12416581809520721, "n": 500 }, "banking77": { "acc": 0.89, "nll": 0.4005417227745056, "ece": 0.06322333976626397, "brier": 0.16635079681873322, "n": 500 }, "emotion": { "acc": 0.894, "nll": 0.38785520195961, "ece": 0.12037511637806893, "brier": 0.1763131022453308, "n": 500 }, "imdb": { "acc": 0.892, "nll": 0.28929969668388367, "ece": 0.023690259099006624, "brier": 0.17336110770702362, "n": 500 }, "mrpc": { "acc": 0.8217054263565892, "nll": 0.4847447872161865, "ece": 0.16866437339967533, "brier": 0.3047117590904236, "n": 258 }, "news20": { "acc": 0.678, "nll": 1.0626460313796997, "ece": 0.05303302364051344, "brier": 0.44085004925727844, "n": 500 }, "sms": { "acc": 0.34558823529411764, "nll": 1.1533381938934326, "ece": 0.3848064363587137, "brier": 0.8449604511260986, "n": 408 }, "snli": { "acc": 0.538, "nll": 0.9769825339317322, "ece": 0.11520019578933716, "brier": 0.5887723565101624, "n": 500 }, "sst2": { "acc": 0.91, "nll": 0.22101609408855438, "ece": 0.02428796315193177, "brier": 0.1290440559387207, "n": 500 }, "yelp": { "acc": 0.522, "nll": 1.1097854375839233, "ece": 0.09259637647867204, "brier": 0.6057519912719727, "n": 500 }, "_macro": { "acc": 0.7415293661650708, "nll": 0.6344732642173767, "ece": 0.03709241313372004, "brier": 0.35542815923690796 } } }, "held_out": { "cola": { "zero_shot_acc": 0.374, "adapted_acc": { "0": 0.374, "5": 0.652, "10": 0.658, "20": 0.686, "40": 0.686 }, "n": 500 }, "dbpedia": { "zero_shot_acc": 0.312, "adapted_acc": { "0": 0.312, "5": 0.488, "10": 0.586, "20": 0.876, "40": 0.952 }, "n": 500 }, "offensive": { "zero_shot_acc": 0.528, "adapted_acc": { "0": 0.528, "5": 0.674, "10": 0.632, "20": 0.656, "40": 0.694 }, "n": 500 } }, "workflow": { "q_acc": 0.825, "all_correct": 0.43666666666666665, "joint_ece": 0.10074469861705296, "k": 4, "n": 300 }, "speed": { "latency_ms_p50": 11.066616000334761, "latency_ms_p95": 13.510893649527132, "cold_first_call_ms": 173.55092199977662, "throughput_workflows_per_s_bs64": 354.2243768052664, "questions_per_workflow": 3, "device": "Tesla T4" }, "type_safety": { "schema_violations": 0, "note": "outputs are a masked softmax over the declared options; an out-of-schema answer cannot be produced" }, "llm_baseline": { "model": "Qwen/Qwen2.5-0.5B-Instruct", "n": 120, "llm_acc": 0.5833333333333334, "alpha_acc": 0.875, "llm_latency_ms_p50": 82.17461200001708, "llm_latency_ms_p95": 150.7793488504376, "families": [ "sst2", "ag_news", "emotion", "snli" ] }, "frr": { "0.5": { "v0": 0.8416666666666666, "tau": 0.83375, "T_first": 4250, "T_re": 1, "FRR": 4250.0, "L_FRR": 0.00023529411764705883, "censored": false, "trivial": true }, "1": { "v0": 0.8479166666666667, "tau": 0.83375, "T_first": 4250, "T_re": 1, "FRR": 4250.0, "L_FRR": 0.00023529411764705883, "censored": false, "trivial": true }, "2": { "v0": 0.8375, "tau": 0.83375, "T_first": 4250, "T_re": 1, "FRR": 4250.0, "L_FRR": 0.00023529411764705883, "censored": false, "trivial": true }, "4": { "v0": 0.8416666666666668, "tau": 0.83375, "T_first": 4250, "T_re": 1, "FRR": 4250.0, "L_FRR": 0.00023529411764705883, "censored": false, "trivial": true }, "8": { "v0": 0.7979166666666667, "tau": 0.83375, "T_first": 4250, "T_re": 300, "FRR": 14.166666666666666, "L_FRR": 0.07058823529411765, "censored": true, "trivial": false } }, "dmr": { "T_discover": 3250, "T_memorize": 3000, "DMR": 1.0833333333333333 }, "meta": { "0": { "cola": { "T_D": 5, "zero_shot": 0.46, "acc10": 0.665, "final": 0.665 }, "dbpedia": { "T_D": 80, "zero_shot": 0.115, "acc10": 0.045, "final": 0.045 }, "offensive": { "T_D": 80, "zero_shot": 0.645, "acc10": 0.64, "final": 0.64 }, "MGR": 1.0 }, "1000": { "cola": { "T_D": 5, "zero_shot": 0.55, "acc10": 0.635, "final": 0.665 }, "dbpedia": { "T_D": 40, "zero_shot": 0.34, "acc10": 0.525, "final": 0.93 }, "offensive": { "T_D": 40, "zero_shot": 0.64, "acc10": 0.64, "final": 0.745 }, "MGR": 1.5874010519681994 }, "2000": { "cola": { "T_D": 20, "zero_shot": 0.4, "acc10": 0.61, "final": 0.665 }, "dbpedia": { "T_D": 40, "zero_shot": 0.305, "acc10": 0.7, "final": 0.955 }, "offensive": { "T_D": 40, "zero_shot": 0.625, "acc10": 0.69, "final": 0.755 }, "MGR": 1.0 }, "3000": { "cola": { "T_D": 10, "zero_shot": 0.4, "acc10": 0.665, "final": 0.665 }, "dbpedia": { "T_D": 40, "zero_shot": 0.245, "acc10": 0.64, "final": 0.975 }, "offensive": { "T_D": 80, "zero_shot": 0.59, "acc10": 0.685, "final": 0.65 }, "MGR": 1.0 }, "4000": { "cola": { "T_D": 5, "zero_shot": 0.41, "acc10": 0.665, "final": 0.665 }, "dbpedia": { "T_D": 40, "zero_shot": 0.305, "acc10": 0.71, "final": 0.975 }, "offensive": { "T_D": 40, "zero_shot": 0.57, "acc10": 0.625, "final": 0.74 }, "MGR": 1.5874010519681994 }, "5000": { "cola": { "T_D": 5, "zero_shot": 0.4, "acc10": 0.665, "final": 0.665 }, "dbpedia": { "T_D": 40, "zero_shot": 0.355, "acc10": 0.68, "final": 0.945 }, "offensive": { "T_D": 80, "zero_shot": 0.575, "acc10": 0.635, "final": 0.715 }, "MGR": 1.2599210498948732 }, "6000": { "cola": { "T_D": 10, "zero_shot": 0.395, "acc10": 0.665, "final": 0.665 }, "dbpedia": { "T_D": 40, "zero_shot": 0.355, "acc10": 0.695, "final": 0.94 }, "offensive": { "T_D": 40, "zero_shot": 0.555, "acc10": 0.66, "final": 0.77 }, "MGR": 1.2599210498948732 } } }