DECISIONx-1.5B / results /eval_report.json
malleshmadapathi's picture
DECISIONx v2: Qwen2.5-1.5B + LoRA decision model, inference package, eval reports
065ec46 verified
Raw History Blame Contribute Delete
18.9 kB
{
"temperature": {
"choice": 1.0779,
"score": 1.0253,
"null": 1.0253
},
"test": {
"ag_news": {
"type": "choice",
"n": 488,
"accuracy": 0.9303,
"chance": 0.25,
"majority": 0.2705,
"nll": 0.201,
"brier": 0.1047,
"ece": 0.026
},
"ag_news_yn": {
"type": "null",
"n": 312,
"accuracy": 0.9295,
"chance": 0.5,
"majority": 0.6314,
"nll": 0.1621,
"brier": 0.0975,
"ece": 0.021
},
"amazon_stars": {
"type": "score",
"n": 800,
"accuracy": 0.6162,
"chance": 0.2,
"majority": 0.2112,
"nll": 0.8907,
"brier": 0.4995,
"ece": 0.0292,
"mae": 0.5045
},
"anli": {
"type": "choice",
"n": 800,
"accuracy": 0.84,
"chance": 0.4169,
"majority": 0.55,
"nll": 0.4101,
"brier": 0.2414,
"ece": 0.0355
},
"app_reviews": {
"type": "score",
"n": 800,
"accuracy": 0.7175,
"chance": 0.2,
"majority": 0.5763,
"nll": 0.7942,
"brier": 0.4037,
"ece": 0.0362,
"mae": 0.6299
},
"arc": {
"type": "choice",
"n": 218,
"accuracy": 0.8486,
"chance": 0.2504,
"majority": 0.2706,
"nll": 0.4031,
"brier": 0.2142,
"ece": 0.0582
},
"banking77": {
"type": "choice",
"n": 454,
"accuracy": 0.8194,
"chance": 0.013,
"majority": 0.0264,
"nll": 0.5496,
"brier": 0.2576,
"ece": 0.0486
},
"banking77_yn": {
"type": "null",
"n": 227,
"accuracy": 0.978,
"chance": 0.5,
"majority": 0.6344,
"nll": 0.0588,
"brier": 0.0328,
"ece": 0.0089
},
"bitext_category": {
"type": "choice",
"n": 262,
"accuracy": 1.0,
"chance": 0.0909,
"majority": 0.2061,
"nll": 0.0014,
"brier": 0.0002,
"ece": 0.0013
},
"bitext_category_yn": {
"type": "null",
"n": 146,
"accuracy": 0.9863,
"chance": 0.5,
"majority": 0.5753,
"nll": 0.0435,
"brier": 0.0192,
"ece": 0.0079
},
"bitext_intent": {
"type": "choice",
"n": 252,
"accuracy": 0.9762,
"chance": 0.037,
"majority": 0.0794,
"nll": 0.0729,
"brier": 0.0356,
"ece": 0.0214
},
"bitext_intent_yn": {
"type": "null",
"n": 140,
"accuracy": 0.9929,
"chance": 0.5,
"majority": 0.7643,
"nll": 0.026,
"brier": 0.0132,
"ece": 0.0053
},
"boolq": {
"type": "null",
"n": 651,
"accuracy": 0.8356,
"chance": 0.5,
"majority": 0.6006,
"nll": 0.3745,
"brier": 0.2374,
"ece": 0.0376
},
"clinc_oos": {
"type": "choice",
"n": 535,
"accuracy": 0.9196,
"chance": 0.0066,
"majority": 0.0224,
"nll": 0.2167,
"brier": 0.1107,
"ece": 0.0207
},
"clinc_oos_yn": {
"type": "null",
"n": 265,
"accuracy": 0.9962,
"chance": 0.5,
"majority": 0.5132,
"nll": 0.0189,
"brier": 0.0088,
"ece": 0.0114
},
"commonlit": {
"type": "score",
"n": 258,
"accuracy": 0.4574,
"chance": 0.2,
"majority": 0.2519,
"nll": 1.1151,
"brier": 0.6129,
"ece": 0.0607,
"mae": 0.6485
},
"commonsense_qa": {
"type": "choice",
"n": 689,
"accuracy": 0.7808,
"chance": 0.2,
"majority": 0.2163,
"nll": 0.5728,
"brier": 0.3027,
"ece": 0.0297
},
"emotion": {
"type": "choice",
"n": 517,
"accuracy": 0.9014,
"chance": 0.1667,
"majority": 0.3482,
"nll": 0.1994,
"brier": 0.1212,
"ece": 0.0364
},
"emotion_yn": {
"type": "null",
"n": 283,
"accuracy": 0.9576,
"chance": 0.5,
"majority": 0.5654,
"nll": 0.1266,
"brier": 0.0703,
"ece": 0.0279
},
"fever": {
"type": "choice",
"n": 800,
"accuracy": 0.9038,
"chance": 0.3956,
"majority": 0.5112,
"nll": 0.2559,
"brier": 0.1445,
"ece": 0.0173
},
"hellaswag": {
"type": "choice",
"n": 800,
"accuracy": 0.8812,
"chance": 0.25,
"majority": 0.2562,
"nll": 0.3182,
"brier": 0.1698,
"ece": 0.013
},
"mnli": {
"type": "choice",
"n": 800,
"accuracy": 0.8912,
"chance": 0.4233,
"majority": 0.5275,
"nll": 0.2736,
"brier": 0.157,
"ece": 0.0177
},
"openbookqa": {
"type": "choice",
"n": 325,
"accuracy": 0.8154,
"chance": 0.25,
"majority": 0.2769,
"nll": 0.4333,
"brier": 0.2293,
"ece": 0.0634
},
"paws": {
"type": "null",
"n": 800,
"accuracy": 0.9463,
"chance": 0.5,
"majority": 0.5487,
"nll": 0.1553,
"brier": 0.0874,
"ece": 0.0176
},
"qqp": {
"type": "null",
"n": 800,
"accuracy": 0.8862,
"chance": 0.5,
"majority": 0.665,
"nll": 0.2803,
"brier": 0.1696,
"ece": 0.0329
},
"sick": {
"type": "score",
"n": 633,
"accuracy": 0.684,
"chance": 0.2,
"majority": 0.4107,
"nll": 0.6634,
"brier": 0.4094,
"ece": 0.0552,
"mae": 0.3845
},
"snli": {
"type": "choice",
"n": 800,
"accuracy": 0.9225,
"chance": 0.4198,
"majority": 0.4888,
"nll": 0.2145,
"brier": 0.1184,
"ece": 0.0165
},
"sst2": {
"type": "choice",
"n": 800,
"accuracy": 0.94,
"chance": 0.5,
"majority": 0.5175,
"nll": 0.1648,
"brier": 0.0935,
"ece": 0.0137
},
"sts_more": {
"type": "score",
"n": 453,
"accuracy": 0.6512,
"chance": 0.1667,
"majority": 0.3333,
"nll": 0.8228,
"brier": 0.4691,
"ece": 0.0733,
"mae": 0.4559
},
"stsb": {
"type": "score",
"n": 337,
"accuracy": 0.6053,
"chance": 0.1667,
"majority": 0.273,
"nll": 0.8552,
"brier": 0.5083,
"ece": 0.0609,
"mae": 0.4576
},
"tripadvisor": {
"type": "score",
"n": 800,
"accuracy": 0.6875,
"chance": 0.2,
"majority": 0.5125,
"nll": 0.7356,
"brier": 0.4309,
"ece": 0.0281,
"mae": 0.4326
},
"tweet_hate": {
"type": "null",
"n": 633,
"accuracy": 0.8199,
"chance": 0.5,
"majority": 0.564,
"nll": 0.4128,
"brier": 0.2621,
"ece": 0.0326
},
"tweet_sentiment": {
"type": "choice",
"n": 645,
"accuracy": 0.7922,
"chance": 0.4013,
"majority": 0.3922,
"nll": 0.4332,
"brier": 0.2717,
"ece": 0.0504
},
"tweet_sentiment_yn": {
"type": "null",
"n": 155,
"accuracy": 0.7871,
"chance": 0.5,
"majority": 0.5484,
"nll": 0.4168,
"brier": 0.2763,
"ece": 0.0659
},
"yelp": {
"type": "score",
"n": 800,
"accuracy": 0.7163,
"chance": 0.2,
"majority": 0.2162,
"nll": 0.6784,
"brier": 0.3981,
"ece": 0.0327,
"mae": 0.3788
},
"_macro": {
"accuracy": 0.8404,
"chance": 0.3173,
"majority": 0.4102,
"nll": 0.3815,
"ece": 0.0319
}
},
"test_uncalibrated": {
"ag_news": {
"type": "choice",
"n": 488,
"accuracy": 0.9303,
"chance": 0.25,
"majority": 0.2705,
"nll": 0.1983,
"brier": 0.1041,
"ece": 0.0203
},
"ag_news_yn": {
"type": "null",
"n": 312,
"accuracy": 0.9295,
"chance": 0.5,
"majority": 0.6314,
"nll": 0.1625,
"brier": 0.0978,
"ece": 0.02
},
"amazon_stars": {
"type": "score",
"n": 800,
"accuracy": 0.6162,
"chance": 0.2,
"majority": 0.2112,
"nll": 0.891,
"brier": 0.4995,
"ece": 0.03,
"mae": 0.503
},
"anli": {
"type": "choice",
"n": 800,
"accuracy": 0.84,
"chance": 0.4169,
"majority": 0.55,
"nll": 0.4118,
"brier": 0.242,
"ece": 0.0413
},
"app_reviews": {
"type": "score",
"n": 800,
"accuracy": 0.7175,
"chance": 0.2,
"majority": 0.5763,
"nll": 0.7935,
"brier": 0.4036,
"ece": 0.025,
"mae": 0.6229
},
"arc": {
"type": "choice",
"n": 218,
"accuracy": 0.8486,
"chance": 0.2504,
"majority": 0.2706,
"nll": 0.4019,
"brier": 0.2139,
"ece": 0.0309
},
"banking77": {
"type": "choice",
"n": 454,
"accuracy": 0.8194,
"chance": 0.013,
"majority": 0.0264,
"nll": 0.5565,
"brier": 0.2594,
"ece": 0.0705
},
"banking77_yn": {
"type": "null",
"n": 227,
"accuracy": 0.978,
"chance": 0.5,
"majority": 0.6344,
"nll": 0.0585,
"brier": 0.0328,
"ece": 0.0085
},
"bitext_category": {
"type": "choice",
"n": 262,
"accuracy": 1.0,
"chance": 0.0909,
"majority": 0.2061,
"nll": 0.0011,
"brier": 0.0002,
"ece": 0.001
},
"bitext_category_yn": {
"type": "null",
"n": 146,
"accuracy": 0.9863,
"chance": 0.5,
"majority": 0.5753,
"nll": 0.0435,
"brier": 0.0192,
"ece": 0.0076
},
"bitext_intent": {
"type": "choice",
"n": 252,
"accuracy": 0.9762,
"chance": 0.037,
"majority": 0.0794,
"nll": 0.0757,
"brier": 0.0363,
"ece": 0.0181
},
"bitext_intent_yn": {
"type": "null",
"n": 140,
"accuracy": 0.9929,
"chance": 0.5,
"majority": 0.7643,
"nll": 0.026,
"brier": 0.0132,
"ece": 0.0046
},
"boolq": {
"type": "null",
"n": 651,
"accuracy": 0.8356,
"chance": 0.5,
"majority": 0.6006,
"nll": 0.3761,
"brier": 0.2379,
"ece": 0.0372
},
"clinc_oos": {
"type": "choice",
"n": 535,
"accuracy": 0.9196,
"chance": 0.0066,
"majority": 0.0224,
"nll": 0.2184,
"brier": 0.1112,
"ece": 0.0227
},
"clinc_oos_yn": {
"type": "null",
"n": 265,
"accuracy": 0.9962,
"chance": 0.5,
"majority": 0.5132,
"nll": 0.0183,
"brier": 0.0087,
"ece": 0.0108
},
"commonlit": {
"type": "score",
"n": 258,
"accuracy": 0.4574,
"chance": 0.2,
"majority": 0.2519,
"nll": 1.1153,
"brier": 0.6132,
"ece": 0.0672,
"mae": 0.6469
},
"commonsense_qa": {
"type": "choice",
"n": 689,
"accuracy": 0.7808,
"chance": 0.2,
"majority": 0.2163,
"nll": 0.5742,
"brier": 0.3031,
"ece": 0.0373
},
"emotion": {
"type": "choice",
"n": 517,
"accuracy": 0.9014,
"chance": 0.1667,
"majority": 0.3482,
"nll": 0.2027,
"brier": 0.1226,
"ece": 0.0347
},
"emotion_yn": {
"type": "null",
"n": 283,
"accuracy": 0.9576,
"chance": 0.5,
"majority": 0.5654,
"nll": 0.1273,
"brier": 0.0704,
"ece": 0.0289
},
"fever": {
"type": "choice",
"n": 800,
"accuracy": 0.9038,
"chance": 0.3956,
"majority": 0.5112,
"nll": 0.2582,
"brier": 0.1455,
"ece": 0.0231
},
"hellaswag": {
"type": "choice",
"n": 800,
"accuracy": 0.8812,
"chance": 0.25,
"majority": 0.2562,
"nll": 0.3205,
"brier": 0.1703,
"ece": 0.0304
},
"mnli": {
"type": "choice",
"n": 800,
"accuracy": 0.8912,
"chance": 0.4233,
"majority": 0.5275,
"nll": 0.2739,
"brier": 0.1574,
"ece": 0.0172
},
"openbookqa": {
"type": "choice",
"n": 325,
"accuracy": 0.8154,
"chance": 0.25,
"majority": 0.2769,
"nll": 0.4289,
"brier": 0.2282,
"ece": 0.0523
},
"paws": {
"type": "null",
"n": 800,
"accuracy": 0.9463,
"chance": 0.5,
"majority": 0.5487,
"nll": 0.1552,
"brier": 0.0875,
"ece": 0.0181
},
"qqp": {
"type": "null",
"n": 800,
"accuracy": 0.8862,
"chance": 0.5,
"majority": 0.665,
"nll": 0.2805,
"brier": 0.1694,
"ece": 0.025
},
"sick": {
"type": "score",
"n": 633,
"accuracy": 0.684,
"chance": 0.2,
"majority": 0.4107,
"nll": 0.6643,
"brier": 0.41,
"ece": 0.0511,
"mae": 0.3832
},
"snli": {
"type": "choice",
"n": 800,
"accuracy": 0.9225,
"chance": 0.4198,
"majority": 0.4888,
"nll": 0.2146,
"brier": 0.1185,
"ece": 0.0156
},
"sst2": {
"type": "choice",
"n": 800,
"accuracy": 0.94,
"chance": 0.5,
"majority": 0.5175,
"nll": 0.1664,
"brier": 0.0937,
"ece": 0.015
},
"sts_more": {
"type": "score",
"n": 453,
"accuracy": 0.6512,
"chance": 0.1667,
"majority": 0.3333,
"nll": 0.8228,
"brier": 0.4689,
"ece": 0.0648,
"mae": 0.455
},
"stsb": {
"type": "score",
"n": 337,
"accuracy": 0.6053,
"chance": 0.1667,
"majority": 0.273,
"nll": 0.8565,
"brier": 0.5091,
"ece": 0.0778,
"mae": 0.4572
},
"tripadvisor": {
"type": "score",
"n": 800,
"accuracy": 0.6875,
"chance": 0.2,
"majority": 0.5125,
"nll": 0.7356,
"brier": 0.4309,
"ece": 0.0329,
"mae": 0.4305
},
"tweet_hate": {
"type": "null",
"n": 633,
"accuracy": 0.8199,
"chance": 0.5,
"majority": 0.564,
"nll": 0.4139,
"brier": 0.2625,
"ece": 0.0343
},
"tweet_sentiment": {
"type": "choice",
"n": 645,
"accuracy": 0.7922,
"chance": 0.4013,
"majority": 0.3922,
"nll": 0.4359,
"brier": 0.2735,
"ece": 0.0621
},
"tweet_sentiment_yn": {
"type": "null",
"n": 155,
"accuracy": 0.7871,
"chance": 0.5,
"majority": 0.5484,
"nll": 0.4186,
"brier": 0.2774,
"ece": 0.0693
},
"yelp": {
"type": "score",
"n": 800,
"accuracy": 0.7163,
"chance": 0.2,
"majority": 0.2162,
"nll": 0.6778,
"brier": 0.3979,
"ece": 0.0294,
"mae": 0.3774
},
"_macro": {
"accuracy": 0.8404,
"chance": 0.3173,
"majority": 0.4102,
"nll": 0.3822,
"ece": 0.0324
}
},
"heldout": {
"counterfactual": {
"type": "null",
"n": 600,
"accuracy": 0.9,
"chance": 0.5,
"majority": 0.8883,
"nll": 0.2634,
"brier": 0.1549,
"ece": 0.0294
},
"dbpedia": {
"type": "choice",
"n": 600,
"accuracy": 0.93,
"chance": 0.0714,
"majority": 0.1017,
"nll": 0.2053,
"brier": 0.1013,
"ece": 0.0235
},
"enron_spam": {
"type": "null",
"n": 600,
"accuracy": 0.615,
"chance": 0.5,
"majority": 0.5067,
"nll": 0.7031,
"brier": 0.487,
"ece": 0.2422
},
"fin_tweets": {
"type": "choice",
"n": 600,
"accuracy": 0.8317,
"chance": 0.3333,
"majority": 0.6867,
"nll": 0.5174,
"brier": 0.298,
"ece": 0.1573
},
"massive": {
"type": "choice",
"n": 600,
"accuracy": 0.735,
"chance": 0.0169,
"majority": 0.0717,
"nll": 1.1317,
"brier": 0.3809,
"ece": 0.1136
},
"rotten_tomatoes": {
"type": "null",
"n": 600,
"accuracy": 0.9217,
"chance": 0.5,
"majority": 0.505,
"nll": 0.2313,
"brier": 0.1316,
"ece": 0.0288
},
"sst5": {
"type": "score",
"n": 600,
"accuracy": 0.5067,
"chance": 0.2,
"majority": 0.2967,
"nll": 1.0277,
"brier": 0.6006,
"ece": 0.0662,
"mae": 0.5386
},
"toxic": {
"type": "null",
"n": 600,
"accuracy": 0.8667,
"chance": 0.5,
"majority": 0.9017,
"nll": 0.3263,
"brier": 0.1945,
"ece": 0.09
},
"trec": {
"type": "choice",
"n": 500,
"accuracy": 0.818,
"chance": 0.1667,
"majority": 0.276,
"nll": 0.5322,
"brier": 0.2674,
"ece": 0.0463
},
"_macro": {
"accuracy": 0.7916,
"chance": 0.3098,
"majority": 0.4705,
"nll": 0.5487,
"ece": 0.0886
}
},
"heldout_uncalibrated": {
"counterfactual": {
"type": "null",
"n": 600,
"accuracy": 0.9,
"chance": 0.5,
"majority": 0.8883,
"nll": 0.2625,
"brier": 0.1545,
"ece": 0.0245
},
"dbpedia": {
"type": "choice",
"n": 600,
"accuracy": 0.93,
"chance": 0.0714,
"majority": 0.1017,
"nll": 0.2054,
"brier": 0.1019,
"ece": 0.0183
},
"enron_spam": {
"type": "null",
"n": 600,
"accuracy": 0.615,
"chance": 0.5,
"majority": 0.5067,
"nll": 0.7134,
"brier": 0.4911,
"ece": 0.2448
},
"fin_tweets": {
"type": "choice",
"n": 600,
"accuracy": 0.8317,
"chance": 0.3333,
"majority": 0.6867,
"nll": 0.5023,
"brier": 0.2901,
"ece": 0.1383
},
"massive": {
"type": "choice",
"n": 600,
"accuracy": 0.735,
"chance": 0.0169,
"majority": 0.0717,
"nll": 1.1827,
"brier": 0.3862,
"ece": 0.1292
},
"rotten_tomatoes": {
"type": "null",
"n": 600,
"accuracy": 0.9217,
"chance": 0.5,
"majority": 0.505,
"nll": 0.2325,
"brier": 0.1317,
"ece": 0.0299
},
"sst5": {
"type": "score",
"n": 600,
"accuracy": 0.5067,
"chance": 0.2,
"majority": 0.2967,
"nll": 1.0287,
"brier": 0.6017,
"ece": 0.0703,
"mae": 0.5396
},
"toxic": {
"type": "null",
"n": 600,
"accuracy": 0.8667,
"chance": 0.5,
"majority": 0.9017,
"nll": 0.3232,
"brier": 0.1931,
"ece": 0.0868
},
"trec": {
"type": "choice",
"n": 500,
"accuracy": 0.818,
"chance": 0.1667,
"majority": 0.276,
"nll": 0.5334,
"brier": 0.2668,
"ece": 0.0399
},
"_macro": {
"accuracy": 0.7916,
"chance": 0.3098,
"majority": 0.4705,
"nll": 0.5538,
"ece": 0.0869
}
}
}