{ "rule": "highest mean accuracy of RACE-Net-V over three development sets (A0 evaluation window, SO-101 history split, Panda history split; equal weight); ties within 0.1 point go to the higher synthetic accuracy", "selected": "v3 refine \u00b7 local \u00b7 6k", "references_dev": { "A0": { "race_s_online (new)": { "mean": 0.9115492957746479, "clean": 0.93, "liar_majority": 0.8916799999999999, "worst": 0.64 }, "race_online (new)": { "mean": 0.9034084507042253, "clean": 0.9399999999999998, "liar_majority": 0.8548, "worst": 0.52 }, "majority": { "mean": 0.677887323943662, "clean": 0.93, "liar_majority": 0.47296, "worst": 0.06 }, "receiver_only": { "mean": 0.8079999999999999, "clean": 0.808, "liar_majority": 0.8080000000000002, "worst": 0.64 } }, "SO-101 history": { "race_s_online (new)": { "mean": 0.8067349551856594, "clean": 0.8709090909090909, "liar_majority": 0.7738181818181818, "worst": 0.5636363636363636 }, "race_online (new)": { "mean": 0.7577464788732394, "clean": 0.7963636363636364, "liar_majority": 0.7157818181818181, "worst": 0.4818181818181818 }, "majority": { "mean": 0.5900640204865558, "clean": 0.7727272727272727, "liar_majority": 0.47643636363636366, "worst": 0.16363636363636364 }, "receiver_only": { "mean": 0.6890909090909091, "clean": 0.6890909090909091, "liar_majority": 0.6890909090909091, "worst": 0.5818181818181818 } }, "Panda history": { "race_s_online (new)": { "mean": 0.8277848911651728, "clean": 0.8690909090909091, "liar_majority": 0.8015272727272729, "worst": 0.5 }, "race_online (new)": { "mean": 0.8517029449423816, "clean": 0.9290909090909091, "liar_majority": 0.7714181818181817, "worst": 0.08181818181818182 }, "majority": { "mean": 0.6311139564660692, "clean": 0.8545454545454545, "liar_majority": 0.5038545454545453, "worst": 0.05454545454545454 }, "receiver_only": { "mean": 0.7418181818181817, "clean": 0.7418181818181818, "liar_majority": 0.741818181818182, "worst": 0.509090909090909 } } }, "candidates": [ { "name": "v1 \u00b7 local \u00b7 6k", "generator": "v1", "steps": 6000, "params": 213889, "advisors": [], "synthetic_v1": 0.9339709490537643, "synthetic_v3": 0.9760187491774559, "dev": { "A0": { "mean": 0.9040281690140846, "clean": 0.9200000000000002, "liar_majority": 0.8879199999999999, "worst": 0.64 }, "SO-101 history": { "mean": 0.7682714468629962, "clean": 0.8018181818181818, "liar_majority": 0.748, "worst": 0.5545454545454546 }, "Panda history": { "mean": 0.7941357234314981, "clean": 0.7527272727272727, "liar_majority": 0.789309090909091, "worst": 0.5272727272727272 } }, "dev_pool": { "A0": { "mean": 0.9216619718309857, "clean": 0.9480000000000001, "liar_majority": 0.8917599999999999, "worst": 0.63 }, "SO-101 history": { "mean": 0.7886555697823304, "clean": 0.8418181818181818, "liar_majority": 0.7506181818181819, "worst": 0.5727272727272728 }, "Panda history": { "mean": 0.8382074263764404, "clean": 0.8672727272727272, "liar_majority": 0.8040727272727272, "worst": 0.4727272727272727 } }, "dev_mean": 0.8221451131028595, "dev_pool_mean": 0.8495083226632523, "selected": false, "test": { "SO-101": { "mean": 0.8155441741357233, "clean": 0.8363636363636363, "liar_majority": 0.7950545454545452, "worst": 0.5181818181818182 }, "Panda": { "mean": 0.8180537772087069, "clean": 0.8181818181818181, "liar_majority": 0.804509090909091, "worst": 0.5545454545454546 } }, "test_pool": { "SO-101": { "mean": 0.8174647887323944, "clean": 0.8454545454545455, "liar_majority": 0.7861090909090908, "worst": 0.5454545454545454 }, "Panda": { "mean": 0.8551088348271447, "clean": 0.8963636363636363, "liar_majority": 0.8172363636363635, "worst": 0.5272727272727272 } } }, { "name": "v1 \u00b7 Kaggle \u00b7 15k", "generator": "v1", "steps": 15000, "params": 213889, "advisors": [], "synthetic_v1": 0.9388268515467644, "synthetic_v3": 0.9777916312217713, "dev": { "A0": { "mean": 0.9016338028169015, "clean": 0.914, "liar_majority": 0.8838400000000002, "worst": 0.64 }, "SO-101 history": { "mean": 0.7727272727272727, "clean": 0.8236363636363636, "liar_majority": 0.7479272727272727, "worst": 0.5545454545454546 }, "Panda history": { "mean": 0.785608194622279, "clean": 0.7527272727272727, "liar_majority": 0.7807272727272727, "worst": 0.509090909090909 } }, "dev_pool": { "A0": { "mean": 0.9205915492957745, "clean": 0.95, "liar_majority": 0.89128, "worst": 0.63 }, "SO-101 history": { "mean": 0.7893725992317542, "clean": 0.8545454545454545, "liar_majority": 0.7506181818181819, "worst": 0.5636363636363636 }, "Panda history": { "mean": 0.8389244558258643, "clean": 0.8763636363636363, "liar_majority": 0.8025454545454547, "worst": 0.4636363636363636 } }, "dev_mean": 0.8199897567221511, "dev_pool_mean": 0.8496295347844643, "selected": false, "test": { "SO-101": { "mean": 0.8088860435339309, "clean": 0.8400000000000001, "liar_majority": 0.7827636363636363, "worst": 0.509090909090909 }, "Panda": { "mean": 0.8225608194622279, "clean": 0.8454545454545453, "liar_majority": 0.8055999999999999, "worst": 0.5545454545454546 } }, "test_pool": { "SO-101": { "mean": 0.8137516005121639, "clean": 0.8563636363636362, "liar_majority": 0.781309090909091, "worst": 0.5454545454545454 }, "Panda": { "mean": 0.8541357234314981, "clean": 0.9036363636363637, "liar_majority": 0.8181090909090909, "worst": 0.5272727272727272 } } }, { "name": "v2 \u00b7 local \u00b7 7k", "generator": "v2", "steps": 7000, "params": 213889, "advisors": [], "synthetic_v1": 0.9257052838802338, "synthetic_v3": 0.9671814292669296, "dev": { "A0": { "mean": 0.9017464788732394, "clean": 0.9179999999999999, "liar_majority": 0.8808000000000001, "worst": 0.64 }, "SO-101 history": { "mean": 0.7901664532650448, "clean": 0.8218181818181819, "liar_majority": 0.7630545454545457, "worst": 0.5363636363636364 }, "Panda history": { "mean": 0.8102176696542893, "clean": 0.7527272727272727, "liar_majority": 0.7944000000000001, "worst": 0.4636363636363636 } }, "dev_pool": { "A0": { "mean": 0.9209859154929577, "clean": 0.9480000000000001, "liar_majority": 0.8915199999999999, "worst": 0.63 }, "SO-101 history": { "mean": 0.7981049935979515, "clean": 0.8781818181818182, "liar_majority": 0.7594181818181818, "worst": 0.5545454545454546 }, "Panda history": { "mean": 0.849090909090909, "clean": 0.8781818181818182, "liar_majority": 0.8118545454545455, "worst": 0.509090909090909 } }, "dev_mean": 0.8340435339308577, "dev_pool_mean": 0.8560606060606061, "selected": false, "test": { "SO-101": { "mean": 0.8259667093469911, "clean": 0.8690909090909091, "liar_majority": 0.7957818181818183, "worst": 0.5181818181818182 }, "Panda": { "mean": 0.8353393085787453, "clean": 0.8854545454545455, "liar_majority": 0.8220363636363636, "worst": 0.5454545454545454 } }, "test_pool": { "SO-101": { "mean": 0.8185147247119078, "clean": 0.8636363636363636, "liar_majority": 0.7869090909090908, "worst": 0.5454545454545454 }, "Panda": { "mean": 0.8613572343149809, "clean": 0.9254545454545454, "liar_majority": 0.8270545454545454, "worst": 0.5454545454545454 } } }, { "name": "v3 \u00b7 Kaggle \u00b7 15k \u00b7 seed 0", "generator": "v3", "steps": 15000, "params": 213889, "advisors": [], "synthetic_v1": 0.9308922812342644, "synthetic_v3": 0.9790606796741486, "dev": { "A0": { "mean": 0.8996338028169014, "clean": 0.908, "liar_majority": 0.88832, "worst": 0.65 }, "SO-101 history": { "mean": 0.7985915492957747, "clean": 0.8636363636363636, "liar_majority": 0.7642181818181817, "worst": 0.5363636363636364 }, "Panda history": { "mean": 0.7994110115236874, "clean": 0.7981818181818181, "liar_majority": 0.7926545454545454, "worst": 0.5 } }, "dev_pool": { "A0": { "mean": 0.9233521126760562, "clean": 0.95, "liar_majority": 0.8959999999999999, "worst": 0.63 }, "SO-101 history": { "mean": 0.7980537772087068, "clean": 0.86, "liar_majority": 0.7568, "worst": 0.5727272727272728 }, "Panda history": { "mean": 0.8480921895006404, "clean": 0.8818181818181818, "liar_majority": 0.8121454545454545, "worst": 0.509090909090909 } }, "dev_mean": 0.8325454545454546, "dev_pool_mean": 0.8564993597951345, "selected": false, "test": { "SO-101": { "mean": 0.8377464788732395, "clean": 0.8781818181818182, "liar_majority": 0.8010181818181819, "worst": 0.5454545454545454 }, "Panda": { "mean": 0.8402560819462228, "clean": 0.86, "liar_majority": 0.8127272727272727, "worst": 0.5 } }, "test_pool": { "SO-101": { "mean": 0.8191549295774646, "clean": 0.86, "liar_majority": 0.7866909090909089, "worst": 0.5454545454545454 }, "Panda": { "mean": 0.8611011523687578, "clean": 0.909090909090909, "liar_majority": 0.823490909090909, "worst": 0.5545454545454546 } } }, { "name": "v3 \u00b7 Kaggle \u00b7 15k \u00b7 seed 1", "generator": "v3", "steps": 15000, "params": 213889, "advisors": [], "synthetic_v1": 0.9316497668623924, "synthetic_v3": 0.9786906063556671, "dev": { "A0": { "mean": 0.9072112676056336, "clean": 0.914, "liar_majority": 0.8920800000000001, "worst": 0.62 }, "SO-101 history": { "mean": 0.7929833546734957, "clean": 0.8654545454545455, "liar_majority": 0.7590545454545455, "worst": 0.5545454545454546 }, "Panda history": { "mean": 0.8094238156209987, "clean": 0.8236363636363636, "liar_majority": 0.7936, "worst": 0.5181818181818182 } }, "dev_pool": { "A0": { "mean": 0.9233239436619718, "clean": 0.95, "liar_majority": 0.8951999999999999, "worst": 0.63 }, "SO-101 history": { "mean": 0.7971062740076825, "clean": 0.8581818181818182, "liar_majority": 0.7566545454545456, "worst": 0.5636363636363636 }, "Panda history": { "mean": 0.843124199743918, "clean": 0.8800000000000001, "liar_majority": 0.8089454545454546, "worst": 0.4818181818181818 } }, "dev_mean": 0.8365394793000428, "dev_pool_mean": 0.8545181391378573, "selected": false, "test": { "SO-101": { "mean": 0.8364148527528807, "clean": 0.8745454545454546, "liar_majority": 0.8010181818181819, "worst": 0.5181818181818182 }, "Panda": { "mean": 0.836056338028169, "clean": 0.8472727272727273, "liar_majority": 0.8118545454545455, "worst": 0.5363636363636364 } }, "test_pool": { "SO-101": { "mean": 0.8198207426376442, "clean": 0.8581818181818182, "liar_majority": 0.7876363636363636, "worst": 0.5454545454545454 }, "Panda": { "mean": 0.8594878361075544, "clean": 0.9090909090909092, "liar_majority": 0.8219636363636365, "worst": 0.5545454545454546 } } }, { "name": "v3 large \u00b7 Kaggle \u00b7 9k", "generator": "v3", "steps": 9000, "params": 589729, "advisors": [], "synthetic_v1": 0.9295911759138107, "synthetic_v3": 0.9787556126713752, "dev": { "A0": { "mean": 0.9014929577464788, "clean": 0.9019999999999999, "liar_majority": 0.89032, "worst": 0.65 }, "SO-101 history": { "mean": 0.8070934699103712, "clean": 0.8781818181818182, "liar_majority": 0.7748363636363635, "worst": 0.5181818181818182 }, "Panda history": { "mean": 0.8068117797695261, "clean": 0.7963636363636364, "liar_majority": 0.7887999999999998, "worst": 0.4909090909090909 } }, "dev_pool": { "A0": { "mean": 0.9228169014084507, "clean": 0.95, "liar_majority": 0.89504, "worst": 0.63 }, "SO-101 history": { "mean": 0.8009475032010243, "clean": 0.8618181818181817, "liar_majority": 0.7617454545454545, "worst": 0.5636363636363636 }, "Panda history": { "mean": 0.844507042253521, "clean": 0.8818181818181818, "liar_majority": 0.8088727272727273, "worst": 0.4909090909090909 } }, "dev_mean": 0.8384660691421253, "dev_pool_mean": 0.8560904822876653, "selected": false, "test": { "SO-101": { "mean": 0.8359282970550576, "clean": 0.8709090909090909, "liar_majority": 0.803490909090909, "worst": 0.5181818181818182 }, "Panda": { "mean": 0.8406658130601794, "clean": 0.8527272727272728, "liar_majority": 0.8197818181818183, "worst": 0.5363636363636364 } }, "test_pool": { "SO-101": { "mean": 0.8222791293213829, "clean": 0.8636363636363636, "liar_majority": 0.7882181818181817, "worst": 0.5454545454545454 }, "Panda": { "mean": 0.8585659411011523, "clean": 0.9109090909090909, "liar_majority": 0.8223272727272728, "worst": 0.5545454545454546 } } }, { "name": "v3 refine \u00b7 local \u00b7 6k", "generator": "v3", "steps": 6000, "params": 214145, "advisors": [ "race_s", "race" ], "synthetic_v1": 0.9263101994991303, "synthetic_v3": 0.9782526358962059, "dev": { "A0": { "mean": 0.9152112676056338, "clean": 0.9260000000000002, "liar_majority": 0.8956799999999999, "worst": 0.64 }, "SO-101 history": { "mean": 0.7951344430217671, "clean": 0.8581818181818182, "liar_majority": 0.7585454545454546, "worst": 0.5363636363636364 }, "Panda history": { "mean": 0.8095518565941101, "clean": 0.809090909090909, "liar_majority": 0.7933090909090909, "worst": 0.5 } }, "dev_pool": { "A0": { "mean": 0.9235774647887324, "clean": 0.95, "liar_majority": 0.8951999999999998, "worst": 0.63 }, "SO-101 history": { "mean": 0.7975672215108834, "clean": 0.86, "liar_majority": 0.7581090909090911, "worst": 0.5727272727272728 }, "Panda history": { "mean": 0.8470166453265044, "clean": 0.8872727272727273, "liar_majority": 0.8079272727272726, "worst": 0.5 } }, "dev_mean": 0.8399658557405036, "dev_pool_mean": 0.8560537772087068, "selected": true, "test": { "SO-101": { "mean": 0.8338284250960307, "clean": 0.8818181818181816, "liar_majority": 0.8004363636363637, "worst": 0.5363636363636364 }, "Panda": { "mean": 0.8418950064020486, "clean": 0.86, "liar_majority": 0.815490909090909, "worst": 0.5454545454545454 } }, "test_pool": { "SO-101": { "mean": 0.8202304737516004, "clean": 0.8618181818181817, "liar_majority": 0.7872000000000001, "worst": 0.5454545454545454 }, "Panda": { "mean": 0.8588220230473751, "clean": 0.9054545454545455, "liar_majority": 0.8239272727272727, "worst": 0.5363636363636364 } } } ] }