Hugging Face's logo

areai51
/
propensity-one-lite

Text Ranking
Scikit-learn
Safetensors
English
headline-testing
ab-testing
click-through-rate
research-preview
Model card Files
xet
 Community
propensity-one-lite / eval.json
areai51's picture
areai51
Propensity One lite v0.2.0: Upworthy-only contrast ridge (research preview)
49afa46 verified
Raw History Blame Contribute Delete
6.66 kB
{
"summary": {
"split": "test",
"grouping": "one randomised A/B test = one group",
"groups": {
"ab_test/validation": {
"n_groups": 1689,
"n_ads": 7568,
"hash": "baf058e3121e0061"
},
"ab_test/test": {
"n_groups": 1945,
"n_ads": 8654,
"hash": "d6360136df563ce6"
}
},
"pairacc": 0.5885,
"pairacc_ci95": [
0.5767,
0.6
],
"pairacc_confw": 0.6274,
"top1": 0.345,
"top1_random": 0.2423,
"top1_decisive_tests": {
"top1": 0.4861,
"ci95": [
0.4241,
0.5387
],
"random": 0.2661,
"n_tests": 323,
"conf": 0.9
},
"validation_pairacc": 0.6008,
"same_groups_baselines_pairacc": {
"random": 0.5029,
"tfidf_ridge": 0.5722,
"lgbm_meta": 0.5389
},
"pairwise_calibration": {
"n_pairs": 16200,
"logloss": 0.6583,
"logloss_constant": 0.6931,
"ece15": 0.0153,
"mean_p_preferred_wins": 0.5963,
"observed_preferred_wins": 0.5956
}
},
"report": {
"snapshot": "upworthy_brx-pipe-0.2.0",
"dataset": "upworthy",
"rows": {
"validation": 7568,
"test": 8654
},
"groups": {
"ab_test/validation": {
"n_groups": 1689,
"n_ads": 7568,
"hash": "baf058e3121e0061"
},
"ab_test/test": {
"n_groups": 1945,
"n_ads": 8654,
"hash": "d6360136df563ce6"
}
},
"results": {
"ab_test": {
"propensity_one_lite": {
"validation": {
"n_groups": 1689,
"n_ads": 7568,
"pairacc_confw": 0.6347,
"pairacc_confw_ci95": [
0.6228,
0.6469
],
"pairacc": 0.6008,
"pairacc_ci95": [
0.5884,
0.6124
],
"pairacc_random": 0.5,
"pairacc_lift": 0.2016,
"ndcg3": 0.9134,
"ndcg3_ci95": [
0.9091,
0.9173
],
"ndcg3_random": 0.8711,
"ndcg3_lift": 0.3279,
"top1": 0.3508,
"top1_ci95": [
0.3271,
0.3739
],
"top1_random": 0.2418,
"top1_lift": 0.1438,
"spearman": 0.242,
"spearman_ci95": [
0.2149,
0.2675
],
"spearman_random": 0.0,
"spearman_global": 0.4202,
"calibration": {
"logloss": 0.6829,
"kl": 0.3148,
"logloss_constant_base": 0.6921,
"brier": 0.1253,
"brier_constant_base": 0.1298,
"ece15": 0.0221,
"p_mean": 0.477,
"t_mean": 0.477,
"reliability": [
{
"p_mean": 0.3557,
"t_mean": 0.2969,
"n": 505
},
{
"p_mean": 0.3925,
"t_mean": 0.3691,
"n": 505
},
{
"p_mean": 0.4112,
"t_mean": 0.4231,
"n": 505
},
{
"p_mean": 0.4264,
"t_mean": 0.4424,
"n": 505
},
{
"p_mean": 0.4381,
"t_mean": 0.46,
"n": 505
},
{
"p_mean": 0.4503,
"t_mean": 0.4508,
"n": 505
},
{
"p_mean": 0.4618,
"t_mean": 0.4869,
"n": 505
},
{
"p_mean": 0.4733,
"t_mean": 0.4816,
"n": 505
},
{
"p_mean": 0.4853,
"t_mean": 0.502,
"n": 504
},
{
"p_mean": 0.4981,
"t_mean": 0.5234,
"n": 504
},
{
"p_mean": 0.5117,
"t_mean": 0.534,
"n": 504
},
{
"p_mean": 0.5266,
"t_mean": 0.5444,
"n": 504
},
{
"p_mean": 0.5444,
"t_mean": 0.5295,
"n": 504
},
{
"p_mean": 0.5664,
"t_mean": 0.5411,
"n": 504
},
{
"p_mean": 0.6144,
"t_mean": 0.5709,
"n": 504
}
]
},
"top1_decisive_tests": {
"top1": 0.479,
"ci95": [
0.4231,
0.535
],
"random": 0.2553,
"n_tests": 286,
"conf": 0.9
}
},
"test": {
"n_groups": 1945,
"n_ads": 8654,
"pairacc_confw": 0.6274,
"pairacc_confw_ci95": [
0.6159,
0.64
],
"pairacc": 0.5885,
"pairacc_ci95": [
0.5767,
0.6
],
"pairacc_random": 0.5,
"pairacc_lift": 0.1771,
"ndcg3": 0.9114,
"ndcg3_ci95": [
0.9077,
0.9151
],
"ndcg3_random": 0.8728,
"ndcg3_lift": 0.3031,
"top1": 0.345,
"top1_ci95": [
0.3254,
0.365
],
"top1_random": 0.2423,
"top1_lift": 0.1356,
"spearman": 0.2189,
"spearman_ci95": [
0.1948,
0.2429
],
"spearman_random": 0.0,
"spearman_global": 0.3979,
"calibration": {
"logloss": 0.683,
"kl": 0.3233,
"logloss_constant_base": 0.6922,
"brier": 0.1286,
"brier_constant_base": 0.1331,
"ece15": 0.0178,
"p_mean": 0.4824,
"t_mean": 0.4783,
"reliability": [
{
"p_mean": 0.3607,
"t_mean": 0.3042,
"n": 577
},
{
"p_mean": 0.3955,
"t_mean": 0.3809,
"n": 577
},
{
"p_mean": 0.4147,
"t_mean": 0.4071,
"n": 577
},
{
"p_mean": 0.4296,
"t_mean": 0.4361,
"n": 577
},
{
"p_mean": 0.4424,
"t_mean": 0.4532,
"n": 577
},
{
"p_mean": 0.4547,
"t_mean": 0.4815,
"n": 577
},
{
"p_mean": 0.467,
"t_mean": 0.4829,
"n": 577
},
{
"p_mean": 0.4793,
"t_mean": 0.504,
"n": 577
},
{
"p_mean": 0.4911,
"t_mean": 0.5028,
"n": 577
},
{
"p_mean": 0.5038,
"t_mean": 0.4994,
"n": 577
},
{
"p_mean": 0.5173,
"t_mean": 0.506,
"n": 577
},
{
"p_mean": 0.5322,
"t_mean": 0.5383,
"n": 577
},
{
"p_mean": 0.5506,
"t_mean": 0.5492,
"n": 577
},
{
"p_mean": 0.5743,
"t_mean": 0.5534,
"n": 577
},
{
"p_mean": 0.6233,
"t_mean": 0.5758,
"n": 576
}
]
},
"top1_decisive_tests": {
"top1": 0.4861,
"ci95": [
0.4241,
0.5387
],
"random": 0.2661,
"n_tests": 323,
"conf": 0.9
}
}
}
}
},
"paired": {
"ab_test": {
"primary": "propensity_one_lite"
}
}
}
}