Download selection.json from Nabidnur/RoboRACE-Net: direct link, hf CLI and curl.
- Browser
- Download file 16.2 kB
-
https://huggingface.co/Nabidnur/RoboRACE-Net/resolve/main/selection.json
- Command line
-
hf download hf://Nabidnur/RoboRACE-Net/selection.json
-
curl -L -o selection.json https://huggingface.co/Nabidnur/RoboRACE-Net/resolve/main/selection.json
16.2 kB
| { | |
| "rule": "highest mean accuracy of RACE-Net-V over three development sets (A0 evaluation window, SO-101 history split, Panda history split; equal weight); ties within 0.1 point go to the higher synthetic accuracy", | |
| "selected": "v3 refine \u00b7 local \u00b7 6k", | |
| "references_dev": { | |
| "A0": { | |
| "race_s_online (new)": { | |
| "mean": 0.9115492957746479, | |
| "clean": 0.93, | |
| "liar_majority": 0.8916799999999999, | |
| "worst": 0.64 | |
| }, | |
| "race_online (new)": { | |
| "mean": 0.9034084507042253, | |
| "clean": 0.9399999999999998, | |
| "liar_majority": 0.8548, | |
| "worst": 0.52 | |
| }, | |
| "majority": { | |
| "mean": 0.677887323943662, | |
| "clean": 0.93, | |
| "liar_majority": 0.47296, | |
| "worst": 0.06 | |
| }, | |
| "receiver_only": { | |
| "mean": 0.8079999999999999, | |
| "clean": 0.808, | |
| "liar_majority": 0.8080000000000002, | |
| "worst": 0.64 | |
| } | |
| }, | |
| "SO-101 history": { | |
| "race_s_online (new)": { | |
| "mean": 0.8067349551856594, | |
| "clean": 0.8709090909090909, | |
| "liar_majority": 0.7738181818181818, | |
| "worst": 0.5636363636363636 | |
| }, | |
| "race_online (new)": { | |
| "mean": 0.7577464788732394, | |
| "clean": 0.7963636363636364, | |
| "liar_majority": 0.7157818181818181, | |
| "worst": 0.4818181818181818 | |
| }, | |
| "majority": { | |
| "mean": 0.5900640204865558, | |
| "clean": 0.7727272727272727, | |
| "liar_majority": 0.47643636363636366, | |
| "worst": 0.16363636363636364 | |
| }, | |
| "receiver_only": { | |
| "mean": 0.6890909090909091, | |
| "clean": 0.6890909090909091, | |
| "liar_majority": 0.6890909090909091, | |
| "worst": 0.5818181818181818 | |
| } | |
| }, | |
| "Panda history": { | |
| "race_s_online (new)": { | |
| "mean": 0.8277848911651728, | |
| "clean": 0.8690909090909091, | |
| "liar_majority": 0.8015272727272729, | |
| "worst": 0.5 | |
| }, | |
| "race_online (new)": { | |
| "mean": 0.8517029449423816, | |
| "clean": 0.9290909090909091, | |
| "liar_majority": 0.7714181818181817, | |
| "worst": 0.08181818181818182 | |
| }, | |
| "majority": { | |
| "mean": 0.6311139564660692, | |
| "clean": 0.8545454545454545, | |
| "liar_majority": 0.5038545454545453, | |
| "worst": 0.05454545454545454 | |
| }, | |
| "receiver_only": { | |
| "mean": 0.7418181818181817, | |
| "clean": 0.7418181818181818, | |
| "liar_majority": 0.741818181818182, | |
| "worst": 0.509090909090909 | |
| } | |
| } | |
| }, | |
| "candidates": [ | |
| { | |
| "name": "v1 \u00b7 local \u00b7 6k", | |
| "generator": "v1", | |
| "steps": 6000, | |
| "params": 213889, | |
| "advisors": [], | |
| "synthetic_v1": 0.9339709490537643, | |
| "synthetic_v3": 0.9760187491774559, | |
| "dev": { | |
| "A0": { | |
| "mean": 0.9040281690140846, | |
| "clean": 0.9200000000000002, | |
| "liar_majority": 0.8879199999999999, | |
| "worst": 0.64 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7682714468629962, | |
| "clean": 0.8018181818181818, | |
| "liar_majority": 0.748, | |
| "worst": 0.5545454545454546 | |
| }, | |
| "Panda history": { | |
| "mean": 0.7941357234314981, | |
| "clean": 0.7527272727272727, | |
| "liar_majority": 0.789309090909091, | |
| "worst": 0.5272727272727272 | |
| } | |
| }, | |
| "dev_pool": { | |
| "A0": { | |
| "mean": 0.9216619718309857, | |
| "clean": 0.9480000000000001, | |
| "liar_majority": 0.8917599999999999, | |
| "worst": 0.63 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7886555697823304, | |
| "clean": 0.8418181818181818, | |
| "liar_majority": 0.7506181818181819, | |
| "worst": 0.5727272727272728 | |
| }, | |
| "Panda history": { | |
| "mean": 0.8382074263764404, | |
| "clean": 0.8672727272727272, | |
| "liar_majority": 0.8040727272727272, | |
| "worst": 0.4727272727272727 | |
| } | |
| }, | |
| "dev_mean": 0.8221451131028595, | |
| "dev_pool_mean": 0.8495083226632523, | |
| "selected": false, | |
| "test": { | |
| "SO-101": { | |
| "mean": 0.8155441741357233, | |
| "clean": 0.8363636363636363, | |
| "liar_majority": 0.7950545454545452, | |
| "worst": 0.5181818181818182 | |
| }, | |
| "Panda": { | |
| "mean": 0.8180537772087069, | |
| "clean": 0.8181818181818181, | |
| "liar_majority": 0.804509090909091, | |
| "worst": 0.5545454545454546 | |
| } | |
| }, | |
| "test_pool": { | |
| "SO-101": { | |
| "mean": 0.8174647887323944, | |
| "clean": 0.8454545454545455, | |
| "liar_majority": 0.7861090909090908, | |
| "worst": 0.5454545454545454 | |
| }, | |
| "Panda": { | |
| "mean": 0.8551088348271447, | |
| "clean": 0.8963636363636363, | |
| "liar_majority": 0.8172363636363635, | |
| "worst": 0.5272727272727272 | |
| } | |
| } | |
| }, | |
| { | |
| "name": "v1 \u00b7 Kaggle \u00b7 15k", | |
| "generator": "v1", | |
| "steps": 15000, | |
| "params": 213889, | |
| "advisors": [], | |
| "synthetic_v1": 0.9388268515467644, | |
| "synthetic_v3": 0.9777916312217713, | |
| "dev": { | |
| "A0": { | |
| "mean": 0.9016338028169015, | |
| "clean": 0.914, | |
| "liar_majority": 0.8838400000000002, | |
| "worst": 0.64 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7727272727272727, | |
| "clean": 0.8236363636363636, | |
| "liar_majority": 0.7479272727272727, | |
| "worst": 0.5545454545454546 | |
| }, | |
| "Panda history": { | |
| "mean": 0.785608194622279, | |
| "clean": 0.7527272727272727, | |
| "liar_majority": 0.7807272727272727, | |
| "worst": 0.509090909090909 | |
| } | |
| }, | |
| "dev_pool": { | |
| "A0": { | |
| "mean": 0.9205915492957745, | |
| "clean": 0.95, | |
| "liar_majority": 0.89128, | |
| "worst": 0.63 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7893725992317542, | |
| "clean": 0.8545454545454545, | |
| "liar_majority": 0.7506181818181819, | |
| "worst": 0.5636363636363636 | |
| }, | |
| "Panda history": { | |
| "mean": 0.8389244558258643, | |
| "clean": 0.8763636363636363, | |
| "liar_majority": 0.8025454545454547, | |
| "worst": 0.4636363636363636 | |
| } | |
| }, | |
| "dev_mean": 0.8199897567221511, | |
| "dev_pool_mean": 0.8496295347844643, | |
| "selected": false, | |
| "test": { | |
| "SO-101": { | |
| "mean": 0.8088860435339309, | |
| "clean": 0.8400000000000001, | |
| "liar_majority": 0.7827636363636363, | |
| "worst": 0.509090909090909 | |
| }, | |
| "Panda": { | |
| "mean": 0.8225608194622279, | |
| "clean": 0.8454545454545453, | |
| "liar_majority": 0.8055999999999999, | |
| "worst": 0.5545454545454546 | |
| } | |
| }, | |
| "test_pool": { | |
| "SO-101": { | |
| "mean": 0.8137516005121639, | |
| "clean": 0.8563636363636362, | |
| "liar_majority": 0.781309090909091, | |
| "worst": 0.5454545454545454 | |
| }, | |
| "Panda": { | |
| "mean": 0.8541357234314981, | |
| "clean": 0.9036363636363637, | |
| "liar_majority": 0.8181090909090909, | |
| "worst": 0.5272727272727272 | |
| } | |
| } | |
| }, | |
| { | |
| "name": "v2 \u00b7 local \u00b7 7k", | |
| "generator": "v2", | |
| "steps": 7000, | |
| "params": 213889, | |
| "advisors": [], | |
| "synthetic_v1": 0.9257052838802338, | |
| "synthetic_v3": 0.9671814292669296, | |
| "dev": { | |
| "A0": { | |
| "mean": 0.9017464788732394, | |
| "clean": 0.9179999999999999, | |
| "liar_majority": 0.8808000000000001, | |
| "worst": 0.64 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7901664532650448, | |
| "clean": 0.8218181818181819, | |
| "liar_majority": 0.7630545454545457, | |
| "worst": 0.5363636363636364 | |
| }, | |
| "Panda history": { | |
| "mean": 0.8102176696542893, | |
| "clean": 0.7527272727272727, | |
| "liar_majority": 0.7944000000000001, | |
| "worst": 0.4636363636363636 | |
| } | |
| }, | |
| "dev_pool": { | |
| "A0": { | |
| "mean": 0.9209859154929577, | |
| "clean": 0.9480000000000001, | |
| "liar_majority": 0.8915199999999999, | |
| "worst": 0.63 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7981049935979515, | |
| "clean": 0.8781818181818182, | |
| "liar_majority": 0.7594181818181818, | |
| "worst": 0.5545454545454546 | |
| }, | |
| "Panda history": { | |
| "mean": 0.849090909090909, | |
| "clean": 0.8781818181818182, | |
| "liar_majority": 0.8118545454545455, | |
| "worst": 0.509090909090909 | |
| } | |
| }, | |
| "dev_mean": 0.8340435339308577, | |
| "dev_pool_mean": 0.8560606060606061, | |
| "selected": false, | |
| "test": { | |
| "SO-101": { | |
| "mean": 0.8259667093469911, | |
| "clean": 0.8690909090909091, | |
| "liar_majority": 0.7957818181818183, | |
| "worst": 0.5181818181818182 | |
| }, | |
| "Panda": { | |
| "mean": 0.8353393085787453, | |
| "clean": 0.8854545454545455, | |
| "liar_majority": 0.8220363636363636, | |
| "worst": 0.5454545454545454 | |
| } | |
| }, | |
| "test_pool": { | |
| "SO-101": { | |
| "mean": 0.8185147247119078, | |
| "clean": 0.8636363636363636, | |
| "liar_majority": 0.7869090909090908, | |
| "worst": 0.5454545454545454 | |
| }, | |
| "Panda": { | |
| "mean": 0.8613572343149809, | |
| "clean": 0.9254545454545454, | |
| "liar_majority": 0.8270545454545454, | |
| "worst": 0.5454545454545454 | |
| } | |
| } | |
| }, | |
| { | |
| "name": "v3 \u00b7 Kaggle \u00b7 15k \u00b7 seed 0", | |
| "generator": "v3", | |
| "steps": 15000, | |
| "params": 213889, | |
| "advisors": [], | |
| "synthetic_v1": 0.9308922812342644, | |
| "synthetic_v3": 0.9790606796741486, | |
| "dev": { | |
| "A0": { | |
| "mean": 0.8996338028169014, | |
| "clean": 0.908, | |
| "liar_majority": 0.88832, | |
| "worst": 0.65 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7985915492957747, | |
| "clean": 0.8636363636363636, | |
| "liar_majority": 0.7642181818181817, | |
| "worst": 0.5363636363636364 | |
| }, | |
| "Panda history": { | |
| "mean": 0.7994110115236874, | |
| "clean": 0.7981818181818181, | |
| "liar_majority": 0.7926545454545454, | |
| "worst": 0.5 | |
| } | |
| }, | |
| "dev_pool": { | |
| "A0": { | |
| "mean": 0.9233521126760562, | |
| "clean": 0.95, | |
| "liar_majority": 0.8959999999999999, | |
| "worst": 0.63 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7980537772087068, | |
| "clean": 0.86, | |
| "liar_majority": 0.7568, | |
| "worst": 0.5727272727272728 | |
| }, | |
| "Panda history": { | |
| "mean": 0.8480921895006404, | |
| "clean": 0.8818181818181818, | |
| "liar_majority": 0.8121454545454545, | |
| "worst": 0.509090909090909 | |
| } | |
| }, | |
| "dev_mean": 0.8325454545454546, | |
| "dev_pool_mean": 0.8564993597951345, | |
| "selected": false, | |
| "test": { | |
| "SO-101": { | |
| "mean": 0.8377464788732395, | |
| "clean": 0.8781818181818182, | |
| "liar_majority": 0.8010181818181819, | |
| "worst": 0.5454545454545454 | |
| }, | |
| "Panda": { | |
| "mean": 0.8402560819462228, | |
| "clean": 0.86, | |
| "liar_majority": 0.8127272727272727, | |
| "worst": 0.5 | |
| } | |
| }, | |
| "test_pool": { | |
| "SO-101": { | |
| "mean": 0.8191549295774646, | |
| "clean": 0.86, | |
| "liar_majority": 0.7866909090909089, | |
| "worst": 0.5454545454545454 | |
| }, | |
| "Panda": { | |
| "mean": 0.8611011523687578, | |
| "clean": 0.909090909090909, | |
| "liar_majority": 0.823490909090909, | |
| "worst": 0.5545454545454546 | |
| } | |
| } | |
| }, | |
| { | |
| "name": "v3 \u00b7 Kaggle \u00b7 15k \u00b7 seed 1", | |
| "generator": "v3", | |
| "steps": 15000, | |
| "params": 213889, | |
| "advisors": [], | |
| "synthetic_v1": 0.9316497668623924, | |
| "synthetic_v3": 0.9786906063556671, | |
| "dev": { | |
| "A0": { | |
| "mean": 0.9072112676056336, | |
| "clean": 0.914, | |
| "liar_majority": 0.8920800000000001, | |
| "worst": 0.62 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7929833546734957, | |
| "clean": 0.8654545454545455, | |
| "liar_majority": 0.7590545454545455, | |
| "worst": 0.5545454545454546 | |
| }, | |
| "Panda history": { | |
| "mean": 0.8094238156209987, | |
| "clean": 0.8236363636363636, | |
| "liar_majority": 0.7936, | |
| "worst": 0.5181818181818182 | |
| } | |
| }, | |
| "dev_pool": { | |
| "A0": { | |
| "mean": 0.9233239436619718, | |
| "clean": 0.95, | |
| "liar_majority": 0.8951999999999999, | |
| "worst": 0.63 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7971062740076825, | |
| "clean": 0.8581818181818182, | |
| "liar_majority": 0.7566545454545456, | |
| "worst": 0.5636363636363636 | |
| }, | |
| "Panda history": { | |
| "mean": 0.843124199743918, | |
| "clean": 0.8800000000000001, | |
| "liar_majority": 0.8089454545454546, | |
| "worst": 0.4818181818181818 | |
| } | |
| }, | |
| "dev_mean": 0.8365394793000428, | |
| "dev_pool_mean": 0.8545181391378573, | |
| "selected": false, | |
| "test": { | |
| "SO-101": { | |
| "mean": 0.8364148527528807, | |
| "clean": 0.8745454545454546, | |
| "liar_majority": 0.8010181818181819, | |
| "worst": 0.5181818181818182 | |
| }, | |
| "Panda": { | |
| "mean": 0.836056338028169, | |
| "clean": 0.8472727272727273, | |
| "liar_majority": 0.8118545454545455, | |
| "worst": 0.5363636363636364 | |
| } | |
| }, | |
| "test_pool": { | |
| "SO-101": { | |
| "mean": 0.8198207426376442, | |
| "clean": 0.8581818181818182, | |
| "liar_majority": 0.7876363636363636, | |
| "worst": 0.5454545454545454 | |
| }, | |
| "Panda": { | |
| "mean": 0.8594878361075544, | |
| "clean": 0.9090909090909092, | |
| "liar_majority": 0.8219636363636365, | |
| "worst": 0.5545454545454546 | |
| } | |
| } | |
| }, | |
| { | |
| "name": "v3 large \u00b7 Kaggle \u00b7 9k", | |
| "generator": "v3", | |
| "steps": 9000, | |
| "params": 589729, | |
| "advisors": [], | |
| "synthetic_v1": 0.9295911759138107, | |
| "synthetic_v3": 0.9787556126713752, | |
| "dev": { | |
| "A0": { | |
| "mean": 0.9014929577464788, | |
| "clean": 0.9019999999999999, | |
| "liar_majority": 0.89032, | |
| "worst": 0.65 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.8070934699103712, | |
| "clean": 0.8781818181818182, | |
| "liar_majority": 0.7748363636363635, | |
| "worst": 0.5181818181818182 | |
| }, | |
| "Panda history": { | |
| "mean": 0.8068117797695261, | |
| "clean": 0.7963636363636364, | |
| "liar_majority": 0.7887999999999998, | |
| "worst": 0.4909090909090909 | |
| } | |
| }, | |
| "dev_pool": { | |
| "A0": { | |
| "mean": 0.9228169014084507, | |
| "clean": 0.95, | |
| "liar_majority": 0.89504, | |
| "worst": 0.63 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.8009475032010243, | |
| "clean": 0.8618181818181817, | |
| "liar_majority": 0.7617454545454545, | |
| "worst": 0.5636363636363636 | |
| }, | |
| "Panda history": { | |
| "mean": 0.844507042253521, | |
| "clean": 0.8818181818181818, | |
| "liar_majority": 0.8088727272727273, | |
| "worst": 0.4909090909090909 | |
| } | |
| }, | |
| "dev_mean": 0.8384660691421253, | |
| "dev_pool_mean": 0.8560904822876653, | |
| "selected": false, | |
| "test": { | |
| "SO-101": { | |
| "mean": 0.8359282970550576, | |
| "clean": 0.8709090909090909, | |
| "liar_majority": 0.803490909090909, | |
| "worst": 0.5181818181818182 | |
| }, | |
| "Panda": { | |
| "mean": 0.8406658130601794, | |
| "clean": 0.8527272727272728, | |
| "liar_majority": 0.8197818181818183, | |
| "worst": 0.5363636363636364 | |
| } | |
| }, | |
| "test_pool": { | |
| "SO-101": { | |
| "mean": 0.8222791293213829, | |
| "clean": 0.8636363636363636, | |
| "liar_majority": 0.7882181818181817, | |
| "worst": 0.5454545454545454 | |
| }, | |
| "Panda": { | |
| "mean": 0.8585659411011523, | |
| "clean": 0.9109090909090909, | |
| "liar_majority": 0.8223272727272728, | |
| "worst": 0.5545454545454546 | |
| } | |
| } | |
| }, | |
| { | |
| "name": "v3 refine \u00b7 local \u00b7 6k", | |
| "generator": "v3", | |
| "steps": 6000, | |
| "params": 214145, | |
| "advisors": [ | |
| "race_s", | |
| "race" | |
| ], | |
| "synthetic_v1": 0.9263101994991303, | |
| "synthetic_v3": 0.9782526358962059, | |
| "dev": { | |
| "A0": { | |
| "mean": 0.9152112676056338, | |
| "clean": 0.9260000000000002, | |
| "liar_majority": 0.8956799999999999, | |
| "worst": 0.64 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7951344430217671, | |
| "clean": 0.8581818181818182, | |
| "liar_majority": 0.7585454545454546, | |
| "worst": 0.5363636363636364 | |
| }, | |
| "Panda history": { | |
| "mean": 0.8095518565941101, | |
| "clean": 0.809090909090909, | |
| "liar_majority": 0.7933090909090909, | |
| "worst": 0.5 | |
| } | |
| }, | |
| "dev_pool": { | |
| "A0": { | |
| "mean": 0.9235774647887324, | |
| "clean": 0.95, | |
| "liar_majority": 0.8951999999999998, | |
| "worst": 0.63 | |
| }, | |
| "SO-101 history": { | |
| "mean": 0.7975672215108834, | |
| "clean": 0.86, | |
| "liar_majority": 0.7581090909090911, | |
| "worst": 0.5727272727272728 | |
| }, | |
| "Panda history": { | |
| "mean": 0.8470166453265044, | |
| "clean": 0.8872727272727273, | |
| "liar_majority": 0.8079272727272726, | |
| "worst": 0.5 | |
| } | |
| }, | |
| "dev_mean": 0.8399658557405036, | |
| "dev_pool_mean": 0.8560537772087068, | |
| "selected": true, | |
| "test": { | |
| "SO-101": { | |
| "mean": 0.8338284250960307, | |
| "clean": 0.8818181818181816, | |
| "liar_majority": 0.8004363636363637, | |
| "worst": 0.5363636363636364 | |
| }, | |
| "Panda": { | |
| "mean": 0.8418950064020486, | |
| "clean": 0.86, | |
| "liar_majority": 0.815490909090909, | |
| "worst": 0.5454545454545454 | |
| } | |
| }, | |
| "test_pool": { | |
| "SO-101": { | |
| "mean": 0.8202304737516004, | |
| "clean": 0.8618181818181817, | |
| "liar_majority": 0.7872000000000001, | |
| "worst": 0.5454545454545454 | |
| }, | |
| "Panda": { | |
| "mean": 0.8588220230473751, | |
| "clean": 0.9054545454545455, | |
| "liar_majority": 0.8239272727272727, | |
| "worst": 0.5363636363636364 | |
| } | |
| } | |
| } | |
| ] | |
| } |