Text Classification
PEFT
Safetensors
English
decision-model
zero-shot-classification
calibration
lora
Instructions to use thinkingdbx/DECISIONx-1.5B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use thinkingdbx/DECISIONx-1.5B with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
Download results/eval_report.json from thinkingdbx/DECISIONx-1.5B: direct link, hf CLI and curl.
- Browser
- Download file 18.9 kB
-
https://huggingface.co/thinkingdbx/DECISIONx-1.5B/resolve/main/results/eval_report.json
- Command line
-
hf download hf://thinkingdbx/DECISIONx-1.5B/results/eval_report.json
-
curl -L -o eval_report.json https://huggingface.co/thinkingdbx/DECISIONx-1.5B/resolve/main/results/eval_report.json
18.9 kB
| { | |
| "temperature": { | |
| "choice": 1.0779, | |
| "score": 1.0253, | |
| "null": 1.0253 | |
| }, | |
| "test": { | |
| "ag_news": { | |
| "type": "choice", | |
| "n": 488, | |
| "accuracy": 0.9303, | |
| "chance": 0.25, | |
| "majority": 0.2705, | |
| "nll": 0.201, | |
| "brier": 0.1047, | |
| "ece": 0.026 | |
| }, | |
| "ag_news_yn": { | |
| "type": "null", | |
| "n": 312, | |
| "accuracy": 0.9295, | |
| "chance": 0.5, | |
| "majority": 0.6314, | |
| "nll": 0.1621, | |
| "brier": 0.0975, | |
| "ece": 0.021 | |
| }, | |
| "amazon_stars": { | |
| "type": "score", | |
| "n": 800, | |
| "accuracy": 0.6162, | |
| "chance": 0.2, | |
| "majority": 0.2112, | |
| "nll": 0.8907, | |
| "brier": 0.4995, | |
| "ece": 0.0292, | |
| "mae": 0.5045 | |
| }, | |
| "anli": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.84, | |
| "chance": 0.4169, | |
| "majority": 0.55, | |
| "nll": 0.4101, | |
| "brier": 0.2414, | |
| "ece": 0.0355 | |
| }, | |
| "app_reviews": { | |
| "type": "score", | |
| "n": 800, | |
| "accuracy": 0.7175, | |
| "chance": 0.2, | |
| "majority": 0.5763, | |
| "nll": 0.7942, | |
| "brier": 0.4037, | |
| "ece": 0.0362, | |
| "mae": 0.6299 | |
| }, | |
| "arc": { | |
| "type": "choice", | |
| "n": 218, | |
| "accuracy": 0.8486, | |
| "chance": 0.2504, | |
| "majority": 0.2706, | |
| "nll": 0.4031, | |
| "brier": 0.2142, | |
| "ece": 0.0582 | |
| }, | |
| "banking77": { | |
| "type": "choice", | |
| "n": 454, | |
| "accuracy": 0.8194, | |
| "chance": 0.013, | |
| "majority": 0.0264, | |
| "nll": 0.5496, | |
| "brier": 0.2576, | |
| "ece": 0.0486 | |
| }, | |
| "banking77_yn": { | |
| "type": "null", | |
| "n": 227, | |
| "accuracy": 0.978, | |
| "chance": 0.5, | |
| "majority": 0.6344, | |
| "nll": 0.0588, | |
| "brier": 0.0328, | |
| "ece": 0.0089 | |
| }, | |
| "bitext_category": { | |
| "type": "choice", | |
| "n": 262, | |
| "accuracy": 1.0, | |
| "chance": 0.0909, | |
| "majority": 0.2061, | |
| "nll": 0.0014, | |
| "brier": 0.0002, | |
| "ece": 0.0013 | |
| }, | |
| "bitext_category_yn": { | |
| "type": "null", | |
| "n": 146, | |
| "accuracy": 0.9863, | |
| "chance": 0.5, | |
| "majority": 0.5753, | |
| "nll": 0.0435, | |
| "brier": 0.0192, | |
| "ece": 0.0079 | |
| }, | |
| "bitext_intent": { | |
| "type": "choice", | |
| "n": 252, | |
| "accuracy": 0.9762, | |
| "chance": 0.037, | |
| "majority": 0.0794, | |
| "nll": 0.0729, | |
| "brier": 0.0356, | |
| "ece": 0.0214 | |
| }, | |
| "bitext_intent_yn": { | |
| "type": "null", | |
| "n": 140, | |
| "accuracy": 0.9929, | |
| "chance": 0.5, | |
| "majority": 0.7643, | |
| "nll": 0.026, | |
| "brier": 0.0132, | |
| "ece": 0.0053 | |
| }, | |
| "boolq": { | |
| "type": "null", | |
| "n": 651, | |
| "accuracy": 0.8356, | |
| "chance": 0.5, | |
| "majority": 0.6006, | |
| "nll": 0.3745, | |
| "brier": 0.2374, | |
| "ece": 0.0376 | |
| }, | |
| "clinc_oos": { | |
| "type": "choice", | |
| "n": 535, | |
| "accuracy": 0.9196, | |
| "chance": 0.0066, | |
| "majority": 0.0224, | |
| "nll": 0.2167, | |
| "brier": 0.1107, | |
| "ece": 0.0207 | |
| }, | |
| "clinc_oos_yn": { | |
| "type": "null", | |
| "n": 265, | |
| "accuracy": 0.9962, | |
| "chance": 0.5, | |
| "majority": 0.5132, | |
| "nll": 0.0189, | |
| "brier": 0.0088, | |
| "ece": 0.0114 | |
| }, | |
| "commonlit": { | |
| "type": "score", | |
| "n": 258, | |
| "accuracy": 0.4574, | |
| "chance": 0.2, | |
| "majority": 0.2519, | |
| "nll": 1.1151, | |
| "brier": 0.6129, | |
| "ece": 0.0607, | |
| "mae": 0.6485 | |
| }, | |
| "commonsense_qa": { | |
| "type": "choice", | |
| "n": 689, | |
| "accuracy": 0.7808, | |
| "chance": 0.2, | |
| "majority": 0.2163, | |
| "nll": 0.5728, | |
| "brier": 0.3027, | |
| "ece": 0.0297 | |
| }, | |
| "emotion": { | |
| "type": "choice", | |
| "n": 517, | |
| "accuracy": 0.9014, | |
| "chance": 0.1667, | |
| "majority": 0.3482, | |
| "nll": 0.1994, | |
| "brier": 0.1212, | |
| "ece": 0.0364 | |
| }, | |
| "emotion_yn": { | |
| "type": "null", | |
| "n": 283, | |
| "accuracy": 0.9576, | |
| "chance": 0.5, | |
| "majority": 0.5654, | |
| "nll": 0.1266, | |
| "brier": 0.0703, | |
| "ece": 0.0279 | |
| }, | |
| "fever": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.9038, | |
| "chance": 0.3956, | |
| "majority": 0.5112, | |
| "nll": 0.2559, | |
| "brier": 0.1445, | |
| "ece": 0.0173 | |
| }, | |
| "hellaswag": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.8812, | |
| "chance": 0.25, | |
| "majority": 0.2562, | |
| "nll": 0.3182, | |
| "brier": 0.1698, | |
| "ece": 0.013 | |
| }, | |
| "mnli": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.8912, | |
| "chance": 0.4233, | |
| "majority": 0.5275, | |
| "nll": 0.2736, | |
| "brier": 0.157, | |
| "ece": 0.0177 | |
| }, | |
| "openbookqa": { | |
| "type": "choice", | |
| "n": 325, | |
| "accuracy": 0.8154, | |
| "chance": 0.25, | |
| "majority": 0.2769, | |
| "nll": 0.4333, | |
| "brier": 0.2293, | |
| "ece": 0.0634 | |
| }, | |
| "paws": { | |
| "type": "null", | |
| "n": 800, | |
| "accuracy": 0.9463, | |
| "chance": 0.5, | |
| "majority": 0.5487, | |
| "nll": 0.1553, | |
| "brier": 0.0874, | |
| "ece": 0.0176 | |
| }, | |
| "qqp": { | |
| "type": "null", | |
| "n": 800, | |
| "accuracy": 0.8862, | |
| "chance": 0.5, | |
| "majority": 0.665, | |
| "nll": 0.2803, | |
| "brier": 0.1696, | |
| "ece": 0.0329 | |
| }, | |
| "sick": { | |
| "type": "score", | |
| "n": 633, | |
| "accuracy": 0.684, | |
| "chance": 0.2, | |
| "majority": 0.4107, | |
| "nll": 0.6634, | |
| "brier": 0.4094, | |
| "ece": 0.0552, | |
| "mae": 0.3845 | |
| }, | |
| "snli": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.9225, | |
| "chance": 0.4198, | |
| "majority": 0.4888, | |
| "nll": 0.2145, | |
| "brier": 0.1184, | |
| "ece": 0.0165 | |
| }, | |
| "sst2": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.94, | |
| "chance": 0.5, | |
| "majority": 0.5175, | |
| "nll": 0.1648, | |
| "brier": 0.0935, | |
| "ece": 0.0137 | |
| }, | |
| "sts_more": { | |
| "type": "score", | |
| "n": 453, | |
| "accuracy": 0.6512, | |
| "chance": 0.1667, | |
| "majority": 0.3333, | |
| "nll": 0.8228, | |
| "brier": 0.4691, | |
| "ece": 0.0733, | |
| "mae": 0.4559 | |
| }, | |
| "stsb": { | |
| "type": "score", | |
| "n": 337, | |
| "accuracy": 0.6053, | |
| "chance": 0.1667, | |
| "majority": 0.273, | |
| "nll": 0.8552, | |
| "brier": 0.5083, | |
| "ece": 0.0609, | |
| "mae": 0.4576 | |
| }, | |
| "tripadvisor": { | |
| "type": "score", | |
| "n": 800, | |
| "accuracy": 0.6875, | |
| "chance": 0.2, | |
| "majority": 0.5125, | |
| "nll": 0.7356, | |
| "brier": 0.4309, | |
| "ece": 0.0281, | |
| "mae": 0.4326 | |
| }, | |
| "tweet_hate": { | |
| "type": "null", | |
| "n": 633, | |
| "accuracy": 0.8199, | |
| "chance": 0.5, | |
| "majority": 0.564, | |
| "nll": 0.4128, | |
| "brier": 0.2621, | |
| "ece": 0.0326 | |
| }, | |
| "tweet_sentiment": { | |
| "type": "choice", | |
| "n": 645, | |
| "accuracy": 0.7922, | |
| "chance": 0.4013, | |
| "majority": 0.3922, | |
| "nll": 0.4332, | |
| "brier": 0.2717, | |
| "ece": 0.0504 | |
| }, | |
| "tweet_sentiment_yn": { | |
| "type": "null", | |
| "n": 155, | |
| "accuracy": 0.7871, | |
| "chance": 0.5, | |
| "majority": 0.5484, | |
| "nll": 0.4168, | |
| "brier": 0.2763, | |
| "ece": 0.0659 | |
| }, | |
| "yelp": { | |
| "type": "score", | |
| "n": 800, | |
| "accuracy": 0.7163, | |
| "chance": 0.2, | |
| "majority": 0.2162, | |
| "nll": 0.6784, | |
| "brier": 0.3981, | |
| "ece": 0.0327, | |
| "mae": 0.3788 | |
| }, | |
| "_macro": { | |
| "accuracy": 0.8404, | |
| "chance": 0.3173, | |
| "majority": 0.4102, | |
| "nll": 0.3815, | |
| "ece": 0.0319 | |
| } | |
| }, | |
| "test_uncalibrated": { | |
| "ag_news": { | |
| "type": "choice", | |
| "n": 488, | |
| "accuracy": 0.9303, | |
| "chance": 0.25, | |
| "majority": 0.2705, | |
| "nll": 0.1983, | |
| "brier": 0.1041, | |
| "ece": 0.0203 | |
| }, | |
| "ag_news_yn": { | |
| "type": "null", | |
| "n": 312, | |
| "accuracy": 0.9295, | |
| "chance": 0.5, | |
| "majority": 0.6314, | |
| "nll": 0.1625, | |
| "brier": 0.0978, | |
| "ece": 0.02 | |
| }, | |
| "amazon_stars": { | |
| "type": "score", | |
| "n": 800, | |
| "accuracy": 0.6162, | |
| "chance": 0.2, | |
| "majority": 0.2112, | |
| "nll": 0.891, | |
| "brier": 0.4995, | |
| "ece": 0.03, | |
| "mae": 0.503 | |
| }, | |
| "anli": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.84, | |
| "chance": 0.4169, | |
| "majority": 0.55, | |
| "nll": 0.4118, | |
| "brier": 0.242, | |
| "ece": 0.0413 | |
| }, | |
| "app_reviews": { | |
| "type": "score", | |
| "n": 800, | |
| "accuracy": 0.7175, | |
| "chance": 0.2, | |
| "majority": 0.5763, | |
| "nll": 0.7935, | |
| "brier": 0.4036, | |
| "ece": 0.025, | |
| "mae": 0.6229 | |
| }, | |
| "arc": { | |
| "type": "choice", | |
| "n": 218, | |
| "accuracy": 0.8486, | |
| "chance": 0.2504, | |
| "majority": 0.2706, | |
| "nll": 0.4019, | |
| "brier": 0.2139, | |
| "ece": 0.0309 | |
| }, | |
| "banking77": { | |
| "type": "choice", | |
| "n": 454, | |
| "accuracy": 0.8194, | |
| "chance": 0.013, | |
| "majority": 0.0264, | |
| "nll": 0.5565, | |
| "brier": 0.2594, | |
| "ece": 0.0705 | |
| }, | |
| "banking77_yn": { | |
| "type": "null", | |
| "n": 227, | |
| "accuracy": 0.978, | |
| "chance": 0.5, | |
| "majority": 0.6344, | |
| "nll": 0.0585, | |
| "brier": 0.0328, | |
| "ece": 0.0085 | |
| }, | |
| "bitext_category": { | |
| "type": "choice", | |
| "n": 262, | |
| "accuracy": 1.0, | |
| "chance": 0.0909, | |
| "majority": 0.2061, | |
| "nll": 0.0011, | |
| "brier": 0.0002, | |
| "ece": 0.001 | |
| }, | |
| "bitext_category_yn": { | |
| "type": "null", | |
| "n": 146, | |
| "accuracy": 0.9863, | |
| "chance": 0.5, | |
| "majority": 0.5753, | |
| "nll": 0.0435, | |
| "brier": 0.0192, | |
| "ece": 0.0076 | |
| }, | |
| "bitext_intent": { | |
| "type": "choice", | |
| "n": 252, | |
| "accuracy": 0.9762, | |
| "chance": 0.037, | |
| "majority": 0.0794, | |
| "nll": 0.0757, | |
| "brier": 0.0363, | |
| "ece": 0.0181 | |
| }, | |
| "bitext_intent_yn": { | |
| "type": "null", | |
| "n": 140, | |
| "accuracy": 0.9929, | |
| "chance": 0.5, | |
| "majority": 0.7643, | |
| "nll": 0.026, | |
| "brier": 0.0132, | |
| "ece": 0.0046 | |
| }, | |
| "boolq": { | |
| "type": "null", | |
| "n": 651, | |
| "accuracy": 0.8356, | |
| "chance": 0.5, | |
| "majority": 0.6006, | |
| "nll": 0.3761, | |
| "brier": 0.2379, | |
| "ece": 0.0372 | |
| }, | |
| "clinc_oos": { | |
| "type": "choice", | |
| "n": 535, | |
| "accuracy": 0.9196, | |
| "chance": 0.0066, | |
| "majority": 0.0224, | |
| "nll": 0.2184, | |
| "brier": 0.1112, | |
| "ece": 0.0227 | |
| }, | |
| "clinc_oos_yn": { | |
| "type": "null", | |
| "n": 265, | |
| "accuracy": 0.9962, | |
| "chance": 0.5, | |
| "majority": 0.5132, | |
| "nll": 0.0183, | |
| "brier": 0.0087, | |
| "ece": 0.0108 | |
| }, | |
| "commonlit": { | |
| "type": "score", | |
| "n": 258, | |
| "accuracy": 0.4574, | |
| "chance": 0.2, | |
| "majority": 0.2519, | |
| "nll": 1.1153, | |
| "brier": 0.6132, | |
| "ece": 0.0672, | |
| "mae": 0.6469 | |
| }, | |
| "commonsense_qa": { | |
| "type": "choice", | |
| "n": 689, | |
| "accuracy": 0.7808, | |
| "chance": 0.2, | |
| "majority": 0.2163, | |
| "nll": 0.5742, | |
| "brier": 0.3031, | |
| "ece": 0.0373 | |
| }, | |
| "emotion": { | |
| "type": "choice", | |
| "n": 517, | |
| "accuracy": 0.9014, | |
| "chance": 0.1667, | |
| "majority": 0.3482, | |
| "nll": 0.2027, | |
| "brier": 0.1226, | |
| "ece": 0.0347 | |
| }, | |
| "emotion_yn": { | |
| "type": "null", | |
| "n": 283, | |
| "accuracy": 0.9576, | |
| "chance": 0.5, | |
| "majority": 0.5654, | |
| "nll": 0.1273, | |
| "brier": 0.0704, | |
| "ece": 0.0289 | |
| }, | |
| "fever": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.9038, | |
| "chance": 0.3956, | |
| "majority": 0.5112, | |
| "nll": 0.2582, | |
| "brier": 0.1455, | |
| "ece": 0.0231 | |
| }, | |
| "hellaswag": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.8812, | |
| "chance": 0.25, | |
| "majority": 0.2562, | |
| "nll": 0.3205, | |
| "brier": 0.1703, | |
| "ece": 0.0304 | |
| }, | |
| "mnli": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.8912, | |
| "chance": 0.4233, | |
| "majority": 0.5275, | |
| "nll": 0.2739, | |
| "brier": 0.1574, | |
| "ece": 0.0172 | |
| }, | |
| "openbookqa": { | |
| "type": "choice", | |
| "n": 325, | |
| "accuracy": 0.8154, | |
| "chance": 0.25, | |
| "majority": 0.2769, | |
| "nll": 0.4289, | |
| "brier": 0.2282, | |
| "ece": 0.0523 | |
| }, | |
| "paws": { | |
| "type": "null", | |
| "n": 800, | |
| "accuracy": 0.9463, | |
| "chance": 0.5, | |
| "majority": 0.5487, | |
| "nll": 0.1552, | |
| "brier": 0.0875, | |
| "ece": 0.0181 | |
| }, | |
| "qqp": { | |
| "type": "null", | |
| "n": 800, | |
| "accuracy": 0.8862, | |
| "chance": 0.5, | |
| "majority": 0.665, | |
| "nll": 0.2805, | |
| "brier": 0.1694, | |
| "ece": 0.025 | |
| }, | |
| "sick": { | |
| "type": "score", | |
| "n": 633, | |
| "accuracy": 0.684, | |
| "chance": 0.2, | |
| "majority": 0.4107, | |
| "nll": 0.6643, | |
| "brier": 0.41, | |
| "ece": 0.0511, | |
| "mae": 0.3832 | |
| }, | |
| "snli": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.9225, | |
| "chance": 0.4198, | |
| "majority": 0.4888, | |
| "nll": 0.2146, | |
| "brier": 0.1185, | |
| "ece": 0.0156 | |
| }, | |
| "sst2": { | |
| "type": "choice", | |
| "n": 800, | |
| "accuracy": 0.94, | |
| "chance": 0.5, | |
| "majority": 0.5175, | |
| "nll": 0.1664, | |
| "brier": 0.0937, | |
| "ece": 0.015 | |
| }, | |
| "sts_more": { | |
| "type": "score", | |
| "n": 453, | |
| "accuracy": 0.6512, | |
| "chance": 0.1667, | |
| "majority": 0.3333, | |
| "nll": 0.8228, | |
| "brier": 0.4689, | |
| "ece": 0.0648, | |
| "mae": 0.455 | |
| }, | |
| "stsb": { | |
| "type": "score", | |
| "n": 337, | |
| "accuracy": 0.6053, | |
| "chance": 0.1667, | |
| "majority": 0.273, | |
| "nll": 0.8565, | |
| "brier": 0.5091, | |
| "ece": 0.0778, | |
| "mae": 0.4572 | |
| }, | |
| "tripadvisor": { | |
| "type": "score", | |
| "n": 800, | |
| "accuracy": 0.6875, | |
| "chance": 0.2, | |
| "majority": 0.5125, | |
| "nll": 0.7356, | |
| "brier": 0.4309, | |
| "ece": 0.0329, | |
| "mae": 0.4305 | |
| }, | |
| "tweet_hate": { | |
| "type": "null", | |
| "n": 633, | |
| "accuracy": 0.8199, | |
| "chance": 0.5, | |
| "majority": 0.564, | |
| "nll": 0.4139, | |
| "brier": 0.2625, | |
| "ece": 0.0343 | |
| }, | |
| "tweet_sentiment": { | |
| "type": "choice", | |
| "n": 645, | |
| "accuracy": 0.7922, | |
| "chance": 0.4013, | |
| "majority": 0.3922, | |
| "nll": 0.4359, | |
| "brier": 0.2735, | |
| "ece": 0.0621 | |
| }, | |
| "tweet_sentiment_yn": { | |
| "type": "null", | |
| "n": 155, | |
| "accuracy": 0.7871, | |
| "chance": 0.5, | |
| "majority": 0.5484, | |
| "nll": 0.4186, | |
| "brier": 0.2774, | |
| "ece": 0.0693 | |
| }, | |
| "yelp": { | |
| "type": "score", | |
| "n": 800, | |
| "accuracy": 0.7163, | |
| "chance": 0.2, | |
| "majority": 0.2162, | |
| "nll": 0.6778, | |
| "brier": 0.3979, | |
| "ece": 0.0294, | |
| "mae": 0.3774 | |
| }, | |
| "_macro": { | |
| "accuracy": 0.8404, | |
| "chance": 0.3173, | |
| "majority": 0.4102, | |
| "nll": 0.3822, | |
| "ece": 0.0324 | |
| } | |
| }, | |
| "heldout": { | |
| "counterfactual": { | |
| "type": "null", | |
| "n": 600, | |
| "accuracy": 0.9, | |
| "chance": 0.5, | |
| "majority": 0.8883, | |
| "nll": 0.2634, | |
| "brier": 0.1549, | |
| "ece": 0.0294 | |
| }, | |
| "dbpedia": { | |
| "type": "choice", | |
| "n": 600, | |
| "accuracy": 0.93, | |
| "chance": 0.0714, | |
| "majority": 0.1017, | |
| "nll": 0.2053, | |
| "brier": 0.1013, | |
| "ece": 0.0235 | |
| }, | |
| "enron_spam": { | |
| "type": "null", | |
| "n": 600, | |
| "accuracy": 0.615, | |
| "chance": 0.5, | |
| "majority": 0.5067, | |
| "nll": 0.7031, | |
| "brier": 0.487, | |
| "ece": 0.2422 | |
| }, | |
| "fin_tweets": { | |
| "type": "choice", | |
| "n": 600, | |
| "accuracy": 0.8317, | |
| "chance": 0.3333, | |
| "majority": 0.6867, | |
| "nll": 0.5174, | |
| "brier": 0.298, | |
| "ece": 0.1573 | |
| }, | |
| "massive": { | |
| "type": "choice", | |
| "n": 600, | |
| "accuracy": 0.735, | |
| "chance": 0.0169, | |
| "majority": 0.0717, | |
| "nll": 1.1317, | |
| "brier": 0.3809, | |
| "ece": 0.1136 | |
| }, | |
| "rotten_tomatoes": { | |
| "type": "null", | |
| "n": 600, | |
| "accuracy": 0.9217, | |
| "chance": 0.5, | |
| "majority": 0.505, | |
| "nll": 0.2313, | |
| "brier": 0.1316, | |
| "ece": 0.0288 | |
| }, | |
| "sst5": { | |
| "type": "score", | |
| "n": 600, | |
| "accuracy": 0.5067, | |
| "chance": 0.2, | |
| "majority": 0.2967, | |
| "nll": 1.0277, | |
| "brier": 0.6006, | |
| "ece": 0.0662, | |
| "mae": 0.5386 | |
| }, | |
| "toxic": { | |
| "type": "null", | |
| "n": 600, | |
| "accuracy": 0.8667, | |
| "chance": 0.5, | |
| "majority": 0.9017, | |
| "nll": 0.3263, | |
| "brier": 0.1945, | |
| "ece": 0.09 | |
| }, | |
| "trec": { | |
| "type": "choice", | |
| "n": 500, | |
| "accuracy": 0.818, | |
| "chance": 0.1667, | |
| "majority": 0.276, | |
| "nll": 0.5322, | |
| "brier": 0.2674, | |
| "ece": 0.0463 | |
| }, | |
| "_macro": { | |
| "accuracy": 0.7916, | |
| "chance": 0.3098, | |
| "majority": 0.4705, | |
| "nll": 0.5487, | |
| "ece": 0.0886 | |
| } | |
| }, | |
| "heldout_uncalibrated": { | |
| "counterfactual": { | |
| "type": "null", | |
| "n": 600, | |
| "accuracy": 0.9, | |
| "chance": 0.5, | |
| "majority": 0.8883, | |
| "nll": 0.2625, | |
| "brier": 0.1545, | |
| "ece": 0.0245 | |
| }, | |
| "dbpedia": { | |
| "type": "choice", | |
| "n": 600, | |
| "accuracy": 0.93, | |
| "chance": 0.0714, | |
| "majority": 0.1017, | |
| "nll": 0.2054, | |
| "brier": 0.1019, | |
| "ece": 0.0183 | |
| }, | |
| "enron_spam": { | |
| "type": "null", | |
| "n": 600, | |
| "accuracy": 0.615, | |
| "chance": 0.5, | |
| "majority": 0.5067, | |
| "nll": 0.7134, | |
| "brier": 0.4911, | |
| "ece": 0.2448 | |
| }, | |
| "fin_tweets": { | |
| "type": "choice", | |
| "n": 600, | |
| "accuracy": 0.8317, | |
| "chance": 0.3333, | |
| "majority": 0.6867, | |
| "nll": 0.5023, | |
| "brier": 0.2901, | |
| "ece": 0.1383 | |
| }, | |
| "massive": { | |
| "type": "choice", | |
| "n": 600, | |
| "accuracy": 0.735, | |
| "chance": 0.0169, | |
| "majority": 0.0717, | |
| "nll": 1.1827, | |
| "brier": 0.3862, | |
| "ece": 0.1292 | |
| }, | |
| "rotten_tomatoes": { | |
| "type": "null", | |
| "n": 600, | |
| "accuracy": 0.9217, | |
| "chance": 0.5, | |
| "majority": 0.505, | |
| "nll": 0.2325, | |
| "brier": 0.1317, | |
| "ece": 0.0299 | |
| }, | |
| "sst5": { | |
| "type": "score", | |
| "n": 600, | |
| "accuracy": 0.5067, | |
| "chance": 0.2, | |
| "majority": 0.2967, | |
| "nll": 1.0287, | |
| "brier": 0.6017, | |
| "ece": 0.0703, | |
| "mae": 0.5396 | |
| }, | |
| "toxic": { | |
| "type": "null", | |
| "n": 600, | |
| "accuracy": 0.8667, | |
| "chance": 0.5, | |
| "majority": 0.9017, | |
| "nll": 0.3232, | |
| "brier": 0.1931, | |
| "ece": 0.0868 | |
| }, | |
| "trec": { | |
| "type": "choice", | |
| "n": 500, | |
| "accuracy": 0.818, | |
| "chance": 0.1667, | |
| "majority": 0.276, | |
| "nll": 0.5334, | |
| "brier": 0.2668, | |
| "ece": 0.0399 | |
| }, | |
| "_macro": { | |
| "accuracy": 0.7916, | |
| "chance": 0.3098, | |
| "majority": 0.4705, | |
| "nll": 0.5538, | |
| "ece": 0.0869 | |
| } | |
| } | |
| } |