Alpha / results.json
SofiTesfay2010's picture
Alpha: System-One-style typed decision model
af6e981 verified
Raw History Blame Contribute Delete
8.14 kB
{
"post_T": 1.0399999618530273,
"quality": {
"train": {
"ag_news": {
"acc": 0.926,
"nll": 0.23406508564949036,
"ece": 0.0323681704998016,
"brier": 0.11545059829950333,
"n": 500
},
"banking77": {
"acc": 0.902,
"nll": 0.3612114489078522,
"ece": 0.035957046896219286,
"brier": 0.15479955077171326,
"n": 500
},
"emotion": {
"acc": 0.922,
"nll": 0.15576711297035217,
"ece": 0.030244563996791818,
"brier": 0.08809012919664383,
"n": 500
},
"imdb": {
"acc": 0.886,
"nll": 0.2880212068557739,
"ece": 0.027435769557952863,
"brier": 0.1726732850074768,
"n": 500
},
"mrpc": {
"acc": 0.8217054263565892,
"nll": 0.3895823657512665,
"ece": 0.042235060486682595,
"brier": 0.25083833932876587,
"n": 258
},
"news20": {
"acc": 0.674,
"nll": 1.0884853601455688,
"ece": 0.06790356469154359,
"brier": 0.4469330608844757,
"n": 500
},
"sms": {
"acc": 0.9926470588235294,
"nll": 0.038313742727041245,
"ece": 0.0037884642096127656,
"brier": 0.015446269884705544,
"n": 408
},
"snli": {
"acc": 0.772,
"nll": 0.5716506838798523,
"ece": 0.045889752805233,
"brier": 0.3297711908817291,
"n": 500
},
"sst2": {
"acc": 0.912,
"nll": 0.24116064608097076,
"ece": 0.034141219377517684,
"brier": 0.14030593633651733,
"n": 500
},
"yelp": {
"acc": 0.572,
"nll": 0.9883294105529785,
"ece": 0.0719212611913681,
"brier": 0.5307314991950989,
"n": 500
},
"_macro": {
"acc": 0.8380352485180118,
"nll": 0.4356587529182434,
"ece": 0.01943733276175781,
"brier": 0.2245039939880371
}
},
"unseen": {
"ag_news": {
"acc": 0.924,
"nll": 0.25852325558662415,
"ece": 0.041514035522937774,
"brier": 0.12416581809520721,
"n": 500
},
"banking77": {
"acc": 0.89,
"nll": 0.4005417227745056,
"ece": 0.06322333976626397,
"brier": 0.16635079681873322,
"n": 500
},
"emotion": {
"acc": 0.894,
"nll": 0.38785520195961,
"ece": 0.12037511637806893,
"brier": 0.1763131022453308,
"n": 500
},
"imdb": {
"acc": 0.892,
"nll": 0.28929969668388367,
"ece": 0.023690259099006624,
"brier": 0.17336110770702362,
"n": 500
},
"mrpc": {
"acc": 0.8217054263565892,
"nll": 0.4847447872161865,
"ece": 0.16866437339967533,
"brier": 0.3047117590904236,
"n": 258
},
"news20": {
"acc": 0.678,
"nll": 1.0626460313796997,
"ece": 0.05303302364051344,
"brier": 0.44085004925727844,
"n": 500
},
"sms": {
"acc": 0.34558823529411764,
"nll": 1.1533381938934326,
"ece": 0.3848064363587137,
"brier": 0.8449604511260986,
"n": 408
},
"snli": {
"acc": 0.538,
"nll": 0.9769825339317322,
"ece": 0.11520019578933716,
"brier": 0.5887723565101624,
"n": 500
},
"sst2": {
"acc": 0.91,
"nll": 0.22101609408855438,
"ece": 0.02428796315193177,
"brier": 0.1290440559387207,
"n": 500
},
"yelp": {
"acc": 0.522,
"nll": 1.1097854375839233,
"ece": 0.09259637647867204,
"brier": 0.6057519912719727,
"n": 500
},
"_macro": {
"acc": 0.7415293661650708,
"nll": 0.6344732642173767,
"ece": 0.03709241313372004,
"brier": 0.35542815923690796
}
}
},
"held_out": {
"cola": {
"zero_shot_acc": 0.374,
"adapted_acc": {
"0": 0.374,
"5": 0.652,
"10": 0.658,
"20": 0.686,
"40": 0.686
},
"n": 500
},
"dbpedia": {
"zero_shot_acc": 0.312,
"adapted_acc": {
"0": 0.312,
"5": 0.488,
"10": 0.586,
"20": 0.876,
"40": 0.952
},
"n": 500
},
"offensive": {
"zero_shot_acc": 0.528,
"adapted_acc": {
"0": 0.528,
"5": 0.674,
"10": 0.632,
"20": 0.656,
"40": 0.694
},
"n": 500
}
},
"workflow": {
"q_acc": 0.825,
"all_correct": 0.43666666666666665,
"joint_ece": 0.10074469861705296,
"k": 4,
"n": 300
},
"speed": {
"latency_ms_p50": 11.066616000334761,
"latency_ms_p95": 13.510893649527132,
"cold_first_call_ms": 173.55092199977662,
"throughput_workflows_per_s_bs64": 354.2243768052664,
"questions_per_workflow": 3,
"device": "Tesla T4"
},
"type_safety": {
"schema_violations": 0,
"note": "outputs are a masked softmax over the declared options; an out-of-schema answer cannot be produced"
},
"llm_baseline": {
"model": "Qwen/Qwen2.5-0.5B-Instruct",
"n": 120,
"llm_acc": 0.5833333333333334,
"alpha_acc": 0.875,
"llm_latency_ms_p50": 82.17461200001708,
"llm_latency_ms_p95": 150.7793488504376,
"families": [
"sst2",
"ag_news",
"emotion",
"snli"
]
},
"frr": {
"0.5": {
"v0": 0.8416666666666666,
"tau": 0.83375,
"T_first": 4250,
"T_re": 1,
"FRR": 4250.0,
"L_FRR": 0.00023529411764705883,
"censored": false,
"trivial": true
},
"1": {
"v0": 0.8479166666666667,
"tau": 0.83375,
"T_first": 4250,
"T_re": 1,
"FRR": 4250.0,
"L_FRR": 0.00023529411764705883,
"censored": false,
"trivial": true
},
"2": {
"v0": 0.8375,
"tau": 0.83375,
"T_first": 4250,
"T_re": 1,
"FRR": 4250.0,
"L_FRR": 0.00023529411764705883,
"censored": false,
"trivial": true
},
"4": {
"v0": 0.8416666666666668,
"tau": 0.83375,
"T_first": 4250,
"T_re": 1,
"FRR": 4250.0,
"L_FRR": 0.00023529411764705883,
"censored": false,
"trivial": true
},
"8": {
"v0": 0.7979166666666667,
"tau": 0.83375,
"T_first": 4250,
"T_re": 300,
"FRR": 14.166666666666666,
"L_FRR": 0.07058823529411765,
"censored": true,
"trivial": false
}
},
"dmr": {
"T_discover": 3250,
"T_memorize": 3000,
"DMR": 1.0833333333333333
},
"meta": {
"0": {
"cola": {
"T_D": 5,
"zero_shot": 0.46,
"acc10": 0.665,
"final": 0.665
},
"dbpedia": {
"T_D": 80,
"zero_shot": 0.115,
"acc10": 0.045,
"final": 0.045
},
"offensive": {
"T_D": 80,
"zero_shot": 0.645,
"acc10": 0.64,
"final": 0.64
},
"MGR": 1.0
},
"1000": {
"cola": {
"T_D": 5,
"zero_shot": 0.55,
"acc10": 0.635,
"final": 0.665
},
"dbpedia": {
"T_D": 40,
"zero_shot": 0.34,
"acc10": 0.525,
"final": 0.93
},
"offensive": {
"T_D": 40,
"zero_shot": 0.64,
"acc10": 0.64,
"final": 0.745
},
"MGR": 1.5874010519681994
},
"2000": {
"cola": {
"T_D": 20,
"zero_shot": 0.4,
"acc10": 0.61,
"final": 0.665
},
"dbpedia": {
"T_D": 40,
"zero_shot": 0.305,
"acc10": 0.7,
"final": 0.955
},
"offensive": {
"T_D": 40,
"zero_shot": 0.625,
"acc10": 0.69,
"final": 0.755
},
"MGR": 1.0
},
"3000": {
"cola": {
"T_D": 10,
"zero_shot": 0.4,
"acc10": 0.665,
"final": 0.665
},
"dbpedia": {
"T_D": 40,
"zero_shot": 0.245,
"acc10": 0.64,
"final": 0.975
},
"offensive": {
"T_D": 80,
"zero_shot": 0.59,
"acc10": 0.685,
"final": 0.65
},
"MGR": 1.0
},
"4000": {
"cola": {
"T_D": 5,
"zero_shot": 0.41,
"acc10": 0.665,
"final": 0.665
},
"dbpedia": {
"T_D": 40,
"zero_shot": 0.305,
"acc10": 0.71,
"final": 0.975
},
"offensive": {
"T_D": 40,
"zero_shot": 0.57,
"acc10": 0.625,
"final": 0.74
},
"MGR": 1.5874010519681994
},
"5000": {
"cola": {
"T_D": 5,
"zero_shot": 0.4,
"acc10": 0.665,
"final": 0.665
},
"dbpedia": {
"T_D": 40,
"zero_shot": 0.355,
"acc10": 0.68,
"final": 0.945
},
"offensive": {
"T_D": 80,
"zero_shot": 0.575,
"acc10": 0.635,
"final": 0.715
},
"MGR": 1.2599210498948732
},
"6000": {
"cola": {
"T_D": 10,
"zero_shot": 0.395,
"acc10": 0.665,
"final": 0.665
},
"dbpedia": {
"T_D": 40,
"zero_shot": 0.355,
"acc10": 0.695,
"final": 0.94
},
"offensive": {
"T_D": 40,
"zero_shot": 0.555,
"acc10": 0.66,
"final": 0.77
},
"MGR": 1.2599210498948732
}
}
}