Download results.json from SofiTesfay2010/Alpha: direct link, hf CLI and curl.
- Browser
- Download file 8.14 kB
-
https://huggingface.co/SofiTesfay2010/Alpha/resolve/main/results.json
- Command line
-
hf download hf://SofiTesfay2010/Alpha/results.json
-
curl -L -o results.json https://huggingface.co/SofiTesfay2010/Alpha/resolve/main/results.json
8.14 kB
| { | |
| "post_T": 1.0399999618530273, | |
| "quality": { | |
| "train": { | |
| "ag_news": { | |
| "acc": 0.926, | |
| "nll": 0.23406508564949036, | |
| "ece": 0.0323681704998016, | |
| "brier": 0.11545059829950333, | |
| "n": 500 | |
| }, | |
| "banking77": { | |
| "acc": 0.902, | |
| "nll": 0.3612114489078522, | |
| "ece": 0.035957046896219286, | |
| "brier": 0.15479955077171326, | |
| "n": 500 | |
| }, | |
| "emotion": { | |
| "acc": 0.922, | |
| "nll": 0.15576711297035217, | |
| "ece": 0.030244563996791818, | |
| "brier": 0.08809012919664383, | |
| "n": 500 | |
| }, | |
| "imdb": { | |
| "acc": 0.886, | |
| "nll": 0.2880212068557739, | |
| "ece": 0.027435769557952863, | |
| "brier": 0.1726732850074768, | |
| "n": 500 | |
| }, | |
| "mrpc": { | |
| "acc": 0.8217054263565892, | |
| "nll": 0.3895823657512665, | |
| "ece": 0.042235060486682595, | |
| "brier": 0.25083833932876587, | |
| "n": 258 | |
| }, | |
| "news20": { | |
| "acc": 0.674, | |
| "nll": 1.0884853601455688, | |
| "ece": 0.06790356469154359, | |
| "brier": 0.4469330608844757, | |
| "n": 500 | |
| }, | |
| "sms": { | |
| "acc": 0.9926470588235294, | |
| "nll": 0.038313742727041245, | |
| "ece": 0.0037884642096127656, | |
| "brier": 0.015446269884705544, | |
| "n": 408 | |
| }, | |
| "snli": { | |
| "acc": 0.772, | |
| "nll": 0.5716506838798523, | |
| "ece": 0.045889752805233, | |
| "brier": 0.3297711908817291, | |
| "n": 500 | |
| }, | |
| "sst2": { | |
| "acc": 0.912, | |
| "nll": 0.24116064608097076, | |
| "ece": 0.034141219377517684, | |
| "brier": 0.14030593633651733, | |
| "n": 500 | |
| }, | |
| "yelp": { | |
| "acc": 0.572, | |
| "nll": 0.9883294105529785, | |
| "ece": 0.0719212611913681, | |
| "brier": 0.5307314991950989, | |
| "n": 500 | |
| }, | |
| "_macro": { | |
| "acc": 0.8380352485180118, | |
| "nll": 0.4356587529182434, | |
| "ece": 0.01943733276175781, | |
| "brier": 0.2245039939880371 | |
| } | |
| }, | |
| "unseen": { | |
| "ag_news": { | |
| "acc": 0.924, | |
| "nll": 0.25852325558662415, | |
| "ece": 0.041514035522937774, | |
| "brier": 0.12416581809520721, | |
| "n": 500 | |
| }, | |
| "banking77": { | |
| "acc": 0.89, | |
| "nll": 0.4005417227745056, | |
| "ece": 0.06322333976626397, | |
| "brier": 0.16635079681873322, | |
| "n": 500 | |
| }, | |
| "emotion": { | |
| "acc": 0.894, | |
| "nll": 0.38785520195961, | |
| "ece": 0.12037511637806893, | |
| "brier": 0.1763131022453308, | |
| "n": 500 | |
| }, | |
| "imdb": { | |
| "acc": 0.892, | |
| "nll": 0.28929969668388367, | |
| "ece": 0.023690259099006624, | |
| "brier": 0.17336110770702362, | |
| "n": 500 | |
| }, | |
| "mrpc": { | |
| "acc": 0.8217054263565892, | |
| "nll": 0.4847447872161865, | |
| "ece": 0.16866437339967533, | |
| "brier": 0.3047117590904236, | |
| "n": 258 | |
| }, | |
| "news20": { | |
| "acc": 0.678, | |
| "nll": 1.0626460313796997, | |
| "ece": 0.05303302364051344, | |
| "brier": 0.44085004925727844, | |
| "n": 500 | |
| }, | |
| "sms": { | |
| "acc": 0.34558823529411764, | |
| "nll": 1.1533381938934326, | |
| "ece": 0.3848064363587137, | |
| "brier": 0.8449604511260986, | |
| "n": 408 | |
| }, | |
| "snli": { | |
| "acc": 0.538, | |
| "nll": 0.9769825339317322, | |
| "ece": 0.11520019578933716, | |
| "brier": 0.5887723565101624, | |
| "n": 500 | |
| }, | |
| "sst2": { | |
| "acc": 0.91, | |
| "nll": 0.22101609408855438, | |
| "ece": 0.02428796315193177, | |
| "brier": 0.1290440559387207, | |
| "n": 500 | |
| }, | |
| "yelp": { | |
| "acc": 0.522, | |
| "nll": 1.1097854375839233, | |
| "ece": 0.09259637647867204, | |
| "brier": 0.6057519912719727, | |
| "n": 500 | |
| }, | |
| "_macro": { | |
| "acc": 0.7415293661650708, | |
| "nll": 0.6344732642173767, | |
| "ece": 0.03709241313372004, | |
| "brier": 0.35542815923690796 | |
| } | |
| } | |
| }, | |
| "held_out": { | |
| "cola": { | |
| "zero_shot_acc": 0.374, | |
| "adapted_acc": { | |
| "0": 0.374, | |
| "5": 0.652, | |
| "10": 0.658, | |
| "20": 0.686, | |
| "40": 0.686 | |
| }, | |
| "n": 500 | |
| }, | |
| "dbpedia": { | |
| "zero_shot_acc": 0.312, | |
| "adapted_acc": { | |
| "0": 0.312, | |
| "5": 0.488, | |
| "10": 0.586, | |
| "20": 0.876, | |
| "40": 0.952 | |
| }, | |
| "n": 500 | |
| }, | |
| "offensive": { | |
| "zero_shot_acc": 0.528, | |
| "adapted_acc": { | |
| "0": 0.528, | |
| "5": 0.674, | |
| "10": 0.632, | |
| "20": 0.656, | |
| "40": 0.694 | |
| }, | |
| "n": 500 | |
| } | |
| }, | |
| "workflow": { | |
| "q_acc": 0.825, | |
| "all_correct": 0.43666666666666665, | |
| "joint_ece": 0.10074469861705296, | |
| "k": 4, | |
| "n": 300 | |
| }, | |
| "speed": { | |
| "latency_ms_p50": 11.066616000334761, | |
| "latency_ms_p95": 13.510893649527132, | |
| "cold_first_call_ms": 173.55092199977662, | |
| "throughput_workflows_per_s_bs64": 354.2243768052664, | |
| "questions_per_workflow": 3, | |
| "device": "Tesla T4" | |
| }, | |
| "type_safety": { | |
| "schema_violations": 0, | |
| "note": "outputs are a masked softmax over the declared options; an out-of-schema answer cannot be produced" | |
| }, | |
| "llm_baseline": { | |
| "model": "Qwen/Qwen2.5-0.5B-Instruct", | |
| "n": 120, | |
| "llm_acc": 0.5833333333333334, | |
| "alpha_acc": 0.875, | |
| "llm_latency_ms_p50": 82.17461200001708, | |
| "llm_latency_ms_p95": 150.7793488504376, | |
| "families": [ | |
| "sst2", | |
| "ag_news", | |
| "emotion", | |
| "snli" | |
| ] | |
| }, | |
| "frr": { | |
| "0.5": { | |
| "v0": 0.8416666666666666, | |
| "tau": 0.83375, | |
| "T_first": 4250, | |
| "T_re": 1, | |
| "FRR": 4250.0, | |
| "L_FRR": 0.00023529411764705883, | |
| "censored": false, | |
| "trivial": true | |
| }, | |
| "1": { | |
| "v0": 0.8479166666666667, | |
| "tau": 0.83375, | |
| "T_first": 4250, | |
| "T_re": 1, | |
| "FRR": 4250.0, | |
| "L_FRR": 0.00023529411764705883, | |
| "censored": false, | |
| "trivial": true | |
| }, | |
| "2": { | |
| "v0": 0.8375, | |
| "tau": 0.83375, | |
| "T_first": 4250, | |
| "T_re": 1, | |
| "FRR": 4250.0, | |
| "L_FRR": 0.00023529411764705883, | |
| "censored": false, | |
| "trivial": true | |
| }, | |
| "4": { | |
| "v0": 0.8416666666666668, | |
| "tau": 0.83375, | |
| "T_first": 4250, | |
| "T_re": 1, | |
| "FRR": 4250.0, | |
| "L_FRR": 0.00023529411764705883, | |
| "censored": false, | |
| "trivial": true | |
| }, | |
| "8": { | |
| "v0": 0.7979166666666667, | |
| "tau": 0.83375, | |
| "T_first": 4250, | |
| "T_re": 300, | |
| "FRR": 14.166666666666666, | |
| "L_FRR": 0.07058823529411765, | |
| "censored": true, | |
| "trivial": false | |
| } | |
| }, | |
| "dmr": { | |
| "T_discover": 3250, | |
| "T_memorize": 3000, | |
| "DMR": 1.0833333333333333 | |
| }, | |
| "meta": { | |
| "0": { | |
| "cola": { | |
| "T_D": 5, | |
| "zero_shot": 0.46, | |
| "acc10": 0.665, | |
| "final": 0.665 | |
| }, | |
| "dbpedia": { | |
| "T_D": 80, | |
| "zero_shot": 0.115, | |
| "acc10": 0.045, | |
| "final": 0.045 | |
| }, | |
| "offensive": { | |
| "T_D": 80, | |
| "zero_shot": 0.645, | |
| "acc10": 0.64, | |
| "final": 0.64 | |
| }, | |
| "MGR": 1.0 | |
| }, | |
| "1000": { | |
| "cola": { | |
| "T_D": 5, | |
| "zero_shot": 0.55, | |
| "acc10": 0.635, | |
| "final": 0.665 | |
| }, | |
| "dbpedia": { | |
| "T_D": 40, | |
| "zero_shot": 0.34, | |
| "acc10": 0.525, | |
| "final": 0.93 | |
| }, | |
| "offensive": { | |
| "T_D": 40, | |
| "zero_shot": 0.64, | |
| "acc10": 0.64, | |
| "final": 0.745 | |
| }, | |
| "MGR": 1.5874010519681994 | |
| }, | |
| "2000": { | |
| "cola": { | |
| "T_D": 20, | |
| "zero_shot": 0.4, | |
| "acc10": 0.61, | |
| "final": 0.665 | |
| }, | |
| "dbpedia": { | |
| "T_D": 40, | |
| "zero_shot": 0.305, | |
| "acc10": 0.7, | |
| "final": 0.955 | |
| }, | |
| "offensive": { | |
| "T_D": 40, | |
| "zero_shot": 0.625, | |
| "acc10": 0.69, | |
| "final": 0.755 | |
| }, | |
| "MGR": 1.0 | |
| }, | |
| "3000": { | |
| "cola": { | |
| "T_D": 10, | |
| "zero_shot": 0.4, | |
| "acc10": 0.665, | |
| "final": 0.665 | |
| }, | |
| "dbpedia": { | |
| "T_D": 40, | |
| "zero_shot": 0.245, | |
| "acc10": 0.64, | |
| "final": 0.975 | |
| }, | |
| "offensive": { | |
| "T_D": 80, | |
| "zero_shot": 0.59, | |
| "acc10": 0.685, | |
| "final": 0.65 | |
| }, | |
| "MGR": 1.0 | |
| }, | |
| "4000": { | |
| "cola": { | |
| "T_D": 5, | |
| "zero_shot": 0.41, | |
| "acc10": 0.665, | |
| "final": 0.665 | |
| }, | |
| "dbpedia": { | |
| "T_D": 40, | |
| "zero_shot": 0.305, | |
| "acc10": 0.71, | |
| "final": 0.975 | |
| }, | |
| "offensive": { | |
| "T_D": 40, | |
| "zero_shot": 0.57, | |
| "acc10": 0.625, | |
| "final": 0.74 | |
| }, | |
| "MGR": 1.5874010519681994 | |
| }, | |
| "5000": { | |
| "cola": { | |
| "T_D": 5, | |
| "zero_shot": 0.4, | |
| "acc10": 0.665, | |
| "final": 0.665 | |
| }, | |
| "dbpedia": { | |
| "T_D": 40, | |
| "zero_shot": 0.355, | |
| "acc10": 0.68, | |
| "final": 0.945 | |
| }, | |
| "offensive": { | |
| "T_D": 80, | |
| "zero_shot": 0.575, | |
| "acc10": 0.635, | |
| "final": 0.715 | |
| }, | |
| "MGR": 1.2599210498948732 | |
| }, | |
| "6000": { | |
| "cola": { | |
| "T_D": 10, | |
| "zero_shot": 0.395, | |
| "acc10": 0.665, | |
| "final": 0.665 | |
| }, | |
| "dbpedia": { | |
| "T_D": 40, | |
| "zero_shot": 0.355, | |
| "acc10": 0.695, | |
| "final": 0.94 | |
| }, | |
| "offensive": { | |
| "T_D": 40, | |
| "zero_shot": 0.555, | |
| "acc10": 0.66, | |
| "final": 0.77 | |
| }, | |
| "MGR": 1.2599210498948732 | |
| } | |
| } | |
| } |