RoboRACE-Net / selection.json
Nabidnur's picture
RACE-Net weights, card and code
6adf6f7 verified
Raw History Blame Contribute Delete
16.2 kB
{
"rule": "highest mean accuracy of RACE-Net-V over three development sets (A0 evaluation window, SO-101 history split, Panda history split; equal weight); ties within 0.1 point go to the higher synthetic accuracy",
"selected": "v3 refine \u00b7 local \u00b7 6k",
"references_dev": {
"A0": {
"race_s_online (new)": {
"mean": 0.9115492957746479,
"clean": 0.93,
"liar_majority": 0.8916799999999999,
"worst": 0.64
},
"race_online (new)": {
"mean": 0.9034084507042253,
"clean": 0.9399999999999998,
"liar_majority": 0.8548,
"worst": 0.52
},
"majority": {
"mean": 0.677887323943662,
"clean": 0.93,
"liar_majority": 0.47296,
"worst": 0.06
},
"receiver_only": {
"mean": 0.8079999999999999,
"clean": 0.808,
"liar_majority": 0.8080000000000002,
"worst": 0.64
}
},
"SO-101 history": {
"race_s_online (new)": {
"mean": 0.8067349551856594,
"clean": 0.8709090909090909,
"liar_majority": 0.7738181818181818,
"worst": 0.5636363636363636
},
"race_online (new)": {
"mean": 0.7577464788732394,
"clean": 0.7963636363636364,
"liar_majority": 0.7157818181818181,
"worst": 0.4818181818181818
},
"majority": {
"mean": 0.5900640204865558,
"clean": 0.7727272727272727,
"liar_majority": 0.47643636363636366,
"worst": 0.16363636363636364
},
"receiver_only": {
"mean": 0.6890909090909091,
"clean": 0.6890909090909091,
"liar_majority": 0.6890909090909091,
"worst": 0.5818181818181818
}
},
"Panda history": {
"race_s_online (new)": {
"mean": 0.8277848911651728,
"clean": 0.8690909090909091,
"liar_majority": 0.8015272727272729,
"worst": 0.5
},
"race_online (new)": {
"mean": 0.8517029449423816,
"clean": 0.9290909090909091,
"liar_majority": 0.7714181818181817,
"worst": 0.08181818181818182
},
"majority": {
"mean": 0.6311139564660692,
"clean": 0.8545454545454545,
"liar_majority": 0.5038545454545453,
"worst": 0.05454545454545454
},
"receiver_only": {
"mean": 0.7418181818181817,
"clean": 0.7418181818181818,
"liar_majority": 0.741818181818182,
"worst": 0.509090909090909
}
}
},
"candidates": [
{
"name": "v1 \u00b7 local \u00b7 6k",
"generator": "v1",
"steps": 6000,
"params": 213889,
"advisors": [],
"synthetic_v1": 0.9339709490537643,
"synthetic_v3": 0.9760187491774559,
"dev": {
"A0": {
"mean": 0.9040281690140846,
"clean": 0.9200000000000002,
"liar_majority": 0.8879199999999999,
"worst": 0.64
},
"SO-101 history": {
"mean": 0.7682714468629962,
"clean": 0.8018181818181818,
"liar_majority": 0.748,
"worst": 0.5545454545454546
},
"Panda history": {
"mean": 0.7941357234314981,
"clean": 0.7527272727272727,
"liar_majority": 0.789309090909091,
"worst": 0.5272727272727272
}
},
"dev_pool": {
"A0": {
"mean": 0.9216619718309857,
"clean": 0.9480000000000001,
"liar_majority": 0.8917599999999999,
"worst": 0.63
},
"SO-101 history": {
"mean": 0.7886555697823304,
"clean": 0.8418181818181818,
"liar_majority": 0.7506181818181819,
"worst": 0.5727272727272728
},
"Panda history": {
"mean": 0.8382074263764404,
"clean": 0.8672727272727272,
"liar_majority": 0.8040727272727272,
"worst": 0.4727272727272727
}
},
"dev_mean": 0.8221451131028595,
"dev_pool_mean": 0.8495083226632523,
"selected": false,
"test": {
"SO-101": {
"mean": 0.8155441741357233,
"clean": 0.8363636363636363,
"liar_majority": 0.7950545454545452,
"worst": 0.5181818181818182
},
"Panda": {
"mean": 0.8180537772087069,
"clean": 0.8181818181818181,
"liar_majority": 0.804509090909091,
"worst": 0.5545454545454546
}
},
"test_pool": {
"SO-101": {
"mean": 0.8174647887323944,
"clean": 0.8454545454545455,
"liar_majority": 0.7861090909090908,
"worst": 0.5454545454545454
},
"Panda": {
"mean": 0.8551088348271447,
"clean": 0.8963636363636363,
"liar_majority": 0.8172363636363635,
"worst": 0.5272727272727272
}
}
},
{
"name": "v1 \u00b7 Kaggle \u00b7 15k",
"generator": "v1",
"steps": 15000,
"params": 213889,
"advisors": [],
"synthetic_v1": 0.9388268515467644,
"synthetic_v3": 0.9777916312217713,
"dev": {
"A0": {
"mean": 0.9016338028169015,
"clean": 0.914,
"liar_majority": 0.8838400000000002,
"worst": 0.64
},
"SO-101 history": {
"mean": 0.7727272727272727,
"clean": 0.8236363636363636,
"liar_majority": 0.7479272727272727,
"worst": 0.5545454545454546
},
"Panda history": {
"mean": 0.785608194622279,
"clean": 0.7527272727272727,
"liar_majority": 0.7807272727272727,
"worst": 0.509090909090909
}
},
"dev_pool": {
"A0": {
"mean": 0.9205915492957745,
"clean": 0.95,
"liar_majority": 0.89128,
"worst": 0.63
},
"SO-101 history": {
"mean": 0.7893725992317542,
"clean": 0.8545454545454545,
"liar_majority": 0.7506181818181819,
"worst": 0.5636363636363636
},
"Panda history": {
"mean": 0.8389244558258643,
"clean": 0.8763636363636363,
"liar_majority": 0.8025454545454547,
"worst": 0.4636363636363636
}
},
"dev_mean": 0.8199897567221511,
"dev_pool_mean": 0.8496295347844643,
"selected": false,
"test": {
"SO-101": {
"mean": 0.8088860435339309,
"clean": 0.8400000000000001,
"liar_majority": 0.7827636363636363,
"worst": 0.509090909090909
},
"Panda": {
"mean": 0.8225608194622279,
"clean": 0.8454545454545453,
"liar_majority": 0.8055999999999999,
"worst": 0.5545454545454546
}
},
"test_pool": {
"SO-101": {
"mean": 0.8137516005121639,
"clean": 0.8563636363636362,
"liar_majority": 0.781309090909091,
"worst": 0.5454545454545454
},
"Panda": {
"mean": 0.8541357234314981,
"clean": 0.9036363636363637,
"liar_majority": 0.8181090909090909,
"worst": 0.5272727272727272
}
}
},
{
"name": "v2 \u00b7 local \u00b7 7k",
"generator": "v2",
"steps": 7000,
"params": 213889,
"advisors": [],
"synthetic_v1": 0.9257052838802338,
"synthetic_v3": 0.9671814292669296,
"dev": {
"A0": {
"mean": 0.9017464788732394,
"clean": 0.9179999999999999,
"liar_majority": 0.8808000000000001,
"worst": 0.64
},
"SO-101 history": {
"mean": 0.7901664532650448,
"clean": 0.8218181818181819,
"liar_majority": 0.7630545454545457,
"worst": 0.5363636363636364
},
"Panda history": {
"mean": 0.8102176696542893,
"clean": 0.7527272727272727,
"liar_majority": 0.7944000000000001,
"worst": 0.4636363636363636
}
},
"dev_pool": {
"A0": {
"mean": 0.9209859154929577,
"clean": 0.9480000000000001,
"liar_majority": 0.8915199999999999,
"worst": 0.63
},
"SO-101 history": {
"mean": 0.7981049935979515,
"clean": 0.8781818181818182,
"liar_majority": 0.7594181818181818,
"worst": 0.5545454545454546
},
"Panda history": {
"mean": 0.849090909090909,
"clean": 0.8781818181818182,
"liar_majority": 0.8118545454545455,
"worst": 0.509090909090909
}
},
"dev_mean": 0.8340435339308577,
"dev_pool_mean": 0.8560606060606061,
"selected": false,
"test": {
"SO-101": {
"mean": 0.8259667093469911,
"clean": 0.8690909090909091,
"liar_majority": 0.7957818181818183,
"worst": 0.5181818181818182
},
"Panda": {
"mean": 0.8353393085787453,
"clean": 0.8854545454545455,
"liar_majority": 0.8220363636363636,
"worst": 0.5454545454545454
}
},
"test_pool": {
"SO-101": {
"mean": 0.8185147247119078,
"clean": 0.8636363636363636,
"liar_majority": 0.7869090909090908,
"worst": 0.5454545454545454
},
"Panda": {
"mean": 0.8613572343149809,
"clean": 0.9254545454545454,
"liar_majority": 0.8270545454545454,
"worst": 0.5454545454545454
}
}
},
{
"name": "v3 \u00b7 Kaggle \u00b7 15k \u00b7 seed 0",
"generator": "v3",
"steps": 15000,
"params": 213889,
"advisors": [],
"synthetic_v1": 0.9308922812342644,
"synthetic_v3": 0.9790606796741486,
"dev": {
"A0": {
"mean": 0.8996338028169014,
"clean": 0.908,
"liar_majority": 0.88832,
"worst": 0.65
},
"SO-101 history": {
"mean": 0.7985915492957747,
"clean": 0.8636363636363636,
"liar_majority": 0.7642181818181817,
"worst": 0.5363636363636364
},
"Panda history": {
"mean": 0.7994110115236874,
"clean": 0.7981818181818181,
"liar_majority": 0.7926545454545454,
"worst": 0.5
}
},
"dev_pool": {
"A0": {
"mean": 0.9233521126760562,
"clean": 0.95,
"liar_majority": 0.8959999999999999,
"worst": 0.63
},
"SO-101 history": {
"mean": 0.7980537772087068,
"clean": 0.86,
"liar_majority": 0.7568,
"worst": 0.5727272727272728
},
"Panda history": {
"mean": 0.8480921895006404,
"clean": 0.8818181818181818,
"liar_majority": 0.8121454545454545,
"worst": 0.509090909090909
}
},
"dev_mean": 0.8325454545454546,
"dev_pool_mean": 0.8564993597951345,
"selected": false,
"test": {
"SO-101": {
"mean": 0.8377464788732395,
"clean": 0.8781818181818182,
"liar_majority": 0.8010181818181819,
"worst": 0.5454545454545454
},
"Panda": {
"mean": 0.8402560819462228,
"clean": 0.86,
"liar_majority": 0.8127272727272727,
"worst": 0.5
}
},
"test_pool": {
"SO-101": {
"mean": 0.8191549295774646,
"clean": 0.86,
"liar_majority": 0.7866909090909089,
"worst": 0.5454545454545454
},
"Panda": {
"mean": 0.8611011523687578,
"clean": 0.909090909090909,
"liar_majority": 0.823490909090909,
"worst": 0.5545454545454546
}
}
},
{
"name": "v3 \u00b7 Kaggle \u00b7 15k \u00b7 seed 1",
"generator": "v3",
"steps": 15000,
"params": 213889,
"advisors": [],
"synthetic_v1": 0.9316497668623924,
"synthetic_v3": 0.9786906063556671,
"dev": {
"A0": {
"mean": 0.9072112676056336,
"clean": 0.914,
"liar_majority": 0.8920800000000001,
"worst": 0.62
},
"SO-101 history": {
"mean": 0.7929833546734957,
"clean": 0.8654545454545455,
"liar_majority": 0.7590545454545455,
"worst": 0.5545454545454546
},
"Panda history": {
"mean": 0.8094238156209987,
"clean": 0.8236363636363636,
"liar_majority": 0.7936,
"worst": 0.5181818181818182
}
},
"dev_pool": {
"A0": {
"mean": 0.9233239436619718,
"clean": 0.95,
"liar_majority": 0.8951999999999999,
"worst": 0.63
},
"SO-101 history": {
"mean": 0.7971062740076825,
"clean": 0.8581818181818182,
"liar_majority": 0.7566545454545456,
"worst": 0.5636363636363636
},
"Panda history": {
"mean": 0.843124199743918,
"clean": 0.8800000000000001,
"liar_majority": 0.8089454545454546,
"worst": 0.4818181818181818
}
},
"dev_mean": 0.8365394793000428,
"dev_pool_mean": 0.8545181391378573,
"selected": false,
"test": {
"SO-101": {
"mean": 0.8364148527528807,
"clean": 0.8745454545454546,
"liar_majority": 0.8010181818181819,
"worst": 0.5181818181818182
},
"Panda": {
"mean": 0.836056338028169,
"clean": 0.8472727272727273,
"liar_majority": 0.8118545454545455,
"worst": 0.5363636363636364
}
},
"test_pool": {
"SO-101": {
"mean": 0.8198207426376442,
"clean": 0.8581818181818182,
"liar_majority": 0.7876363636363636,
"worst": 0.5454545454545454
},
"Panda": {
"mean": 0.8594878361075544,
"clean": 0.9090909090909092,
"liar_majority": 0.8219636363636365,
"worst": 0.5545454545454546
}
}
},
{
"name": "v3 large \u00b7 Kaggle \u00b7 9k",
"generator": "v3",
"steps": 9000,
"params": 589729,
"advisors": [],
"synthetic_v1": 0.9295911759138107,
"synthetic_v3": 0.9787556126713752,
"dev": {
"A0": {
"mean": 0.9014929577464788,
"clean": 0.9019999999999999,
"liar_majority": 0.89032,
"worst": 0.65
},
"SO-101 history": {
"mean": 0.8070934699103712,
"clean": 0.8781818181818182,
"liar_majority": 0.7748363636363635,
"worst": 0.5181818181818182
},
"Panda history": {
"mean": 0.8068117797695261,
"clean": 0.7963636363636364,
"liar_majority": 0.7887999999999998,
"worst": 0.4909090909090909
}
},
"dev_pool": {
"A0": {
"mean": 0.9228169014084507,
"clean": 0.95,
"liar_majority": 0.89504,
"worst": 0.63
},
"SO-101 history": {
"mean": 0.8009475032010243,
"clean": 0.8618181818181817,
"liar_majority": 0.7617454545454545,
"worst": 0.5636363636363636
},
"Panda history": {
"mean": 0.844507042253521,
"clean": 0.8818181818181818,
"liar_majority": 0.8088727272727273,
"worst": 0.4909090909090909
}
},
"dev_mean": 0.8384660691421253,
"dev_pool_mean": 0.8560904822876653,
"selected": false,
"test": {
"SO-101": {
"mean": 0.8359282970550576,
"clean": 0.8709090909090909,
"liar_majority": 0.803490909090909,
"worst": 0.5181818181818182
},
"Panda": {
"mean": 0.8406658130601794,
"clean": 0.8527272727272728,
"liar_majority": 0.8197818181818183,
"worst": 0.5363636363636364
}
},
"test_pool": {
"SO-101": {
"mean": 0.8222791293213829,
"clean": 0.8636363636363636,
"liar_majority": 0.7882181818181817,
"worst": 0.5454545454545454
},
"Panda": {
"mean": 0.8585659411011523,
"clean": 0.9109090909090909,
"liar_majority": 0.8223272727272728,
"worst": 0.5545454545454546
}
}
},
{
"name": "v3 refine \u00b7 local \u00b7 6k",
"generator": "v3",
"steps": 6000,
"params": 214145,
"advisors": [
"race_s",
"race"
],
"synthetic_v1": 0.9263101994991303,
"synthetic_v3": 0.9782526358962059,
"dev": {
"A0": {
"mean": 0.9152112676056338,
"clean": 0.9260000000000002,
"liar_majority": 0.8956799999999999,
"worst": 0.64
},
"SO-101 history": {
"mean": 0.7951344430217671,
"clean": 0.8581818181818182,
"liar_majority": 0.7585454545454546,
"worst": 0.5363636363636364
},
"Panda history": {
"mean": 0.8095518565941101,
"clean": 0.809090909090909,
"liar_majority": 0.7933090909090909,
"worst": 0.5
}
},
"dev_pool": {
"A0": {
"mean": 0.9235774647887324,
"clean": 0.95,
"liar_majority": 0.8951999999999998,
"worst": 0.63
},
"SO-101 history": {
"mean": 0.7975672215108834,
"clean": 0.86,
"liar_majority": 0.7581090909090911,
"worst": 0.5727272727272728
},
"Panda history": {
"mean": 0.8470166453265044,
"clean": 0.8872727272727273,
"liar_majority": 0.8079272727272726,
"worst": 0.5
}
},
"dev_mean": 0.8399658557405036,
"dev_pool_mean": 0.8560537772087068,
"selected": true,
"test": {
"SO-101": {
"mean": 0.8338284250960307,
"clean": 0.8818181818181816,
"liar_majority": 0.8004363636363637,
"worst": 0.5363636363636364
},
"Panda": {
"mean": 0.8418950064020486,
"clean": 0.86,
"liar_majority": 0.815490909090909,
"worst": 0.5454545454545454
}
},
"test_pool": {
"SO-101": {
"mean": 0.8202304737516004,
"clean": 0.8618181818181817,
"liar_majority": 0.7872000000000001,
"worst": 0.5454545454545454
},
"Panda": {
"mean": 0.8588220230473751,
"clean": 0.9054545454545455,
"liar_majority": 0.8239272727272727,
"worst": 0.5363636363636364
}
}
}
]
}