Hugging Face's logo

Spaces:
SelectArena
/
Leaderboard
Running

App Files Community
schaeferbasti commited on
Commit
9d0e14a
·
1 Parent(s): 33f543d

Add links to fs methods

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. data/meta.json +1 -1
  2. data/results_explorer.html +2 -0
  3. data/stability_table.html +3 -1
  4. data/subsets/tasks_all/models_all/budgets_0/leaderboard_table.html +3 -1
  5. data/subsets/tasks_all/models_all/budgets_0/pareto.html +1 -1
  6. data/subsets/tasks_all/models_all/budgets_0/per_model_table.html +2 -0
  7. data/subsets/tasks_all/models_all/budgets_1/leaderboard_table.html +3 -1
  8. data/subsets/tasks_all/models_all/budgets_1/per_model_table.html +2 -0
  9. data/subsets/tasks_all/models_all/budgets_2/leaderboard_table.html +3 -1
  10. data/subsets/tasks_all/models_all/budgets_2/pareto.html +1 -1
  11. data/subsets/tasks_all/models_all/budgets_2/per_model_table.html +2 -0
  12. data/subsets/tasks_all/models_all/budgets_3/leaderboard_table.html +3 -1
  13. data/subsets/tasks_all/models_all/budgets_3/pareto.html +1 -1
  14. data/subsets/tasks_all/models_all/budgets_3/per_model_table.html +2 -0
  15. data/subsets/tasks_all/models_all/budgets_4/leaderboard_table.html +3 -1
  16. data/subsets/tasks_all/models_all/budgets_4/pareto.html +1 -1
  17. data/subsets/tasks_all/models_all/budgets_4/per_model_table.html +2 -0
  18. data/subsets/tasks_all/models_all/budgets_all/leaderboard_table.html +3 -1
  19. data/subsets/tasks_all/models_all/budgets_all/per_model_table.html +2 -0
  20. data/subsets/tasks_all/models_lightgbm/budgets_0/leaderboard_table.html +3 -1
  21. data/subsets/tasks_all/models_lightgbm/budgets_0/pareto.html +1 -1
  22. data/subsets/tasks_all/models_lightgbm/budgets_1/leaderboard_table.html +3 -1
  23. data/subsets/tasks_all/models_lightgbm/budgets_2/leaderboard_table.html +3 -1
  24. data/subsets/tasks_all/models_lightgbm/budgets_3/leaderboard_table.html +3 -1
  25. data/subsets/tasks_all/models_lightgbm/budgets_3/pareto.html +1 -1
  26. data/subsets/tasks_all/models_lightgbm/budgets_4/leaderboard_table.html +3 -1
  27. data/subsets/tasks_all/models_lightgbm/budgets_4/pareto.html +1 -1
  28. data/subsets/tasks_all/models_lightgbm/budgets_all/leaderboard_table.html +3 -1
  29. data/subsets/tasks_all/models_lightgbm/budgets_all/pareto.html +1 -1
  30. data/subsets/tasks_all/models_linearmodel/budgets_0/leaderboard_table.html +3 -1
  31. data/subsets/tasks_all/models_linearmodel/budgets_0/pareto.html +1 -1
  32. data/subsets/tasks_all/models_linearmodel/budgets_1/leaderboard_table.html +3 -1
  33. data/subsets/tasks_all/models_linearmodel/budgets_1/pareto.html +1 -1
  34. data/subsets/tasks_all/models_linearmodel/budgets_2/leaderboard_table.html +3 -1
  35. data/subsets/tasks_all/models_linearmodel/budgets_2/pareto.html +1 -1
  36. data/subsets/tasks_all/models_linearmodel/budgets_3/leaderboard_table.html +3 -1
  37. data/subsets/tasks_all/models_linearmodel/budgets_3/pareto.html +1 -1
  38. data/subsets/tasks_all/models_linearmodel/budgets_4/leaderboard_table.html +3 -1
  39. data/subsets/tasks_all/models_linearmodel/budgets_4/pareto.html +1 -1
  40. data/subsets/tasks_all/models_linearmodel/budgets_all/leaderboard_table.html +3 -1
  41. data/subsets/tasks_all/models_linearmodel/budgets_all/pareto.html +1 -1
  42. data/subsets/tasks_all/models_randomforest/budgets_0/leaderboard_table.html +3 -1
  43. data/subsets/tasks_all/models_randomforest/budgets_0/pareto.html +1 -1
  44. data/subsets/tasks_all/models_randomforest/budgets_1/leaderboard_table.html +3 -1
  45. data/subsets/tasks_all/models_randomforest/budgets_1/pareto.html +1 -1
  46. data/subsets/tasks_all/models_randomforest/budgets_2/leaderboard_table.html +3 -1
  47. data/subsets/tasks_all/models_randomforest/budgets_2/pareto.html +1 -1
  48. data/subsets/tasks_all/models_randomforest/budgets_3/leaderboard_table.html +3 -1
  49. data/subsets/tasks_all/models_randomforest/budgets_3/pareto.html +1 -1
  50. data/subsets/tasks_all/models_randomforest/budgets_4/leaderboard_table.html +3 -1
data/meta.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "benchmark_name": "feature_selection_benchmark_2026_all",
3
- "generated_at": "2026-11-04 20:17 UTC",
4
  "selectarena_version": "0.1.0",
5
  "n_results": 79873,
6
  "n_datasets": 71,
 
1
  {
2
  "benchmark_name": "feature_selection_benchmark_2026_all",
3
+ "generated_at": "2026-11-04 21:54 UTC",
4
  "selectarena_version": "0.1.0",
5
  "n_results": 79873,
6
  "n_datasets": 71,
data/results_explorer.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
data/stability_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort.</span><button onclick="saCsv('selectarena_stability.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class=''>Method</th><th class='num' title='Mean over datasets and budgets.'>Stability ↑</th><th class='num'>Budget 1 (smallest)</th><th class='num'>Budget 2</th><th class='num'>Budget 3</th><th class='num'>Budget 4</th><th class='num'>Budget 5 (largest)</th></tr></thead><tbody><tr><td class='method' data-sort='F-Test'>F-Test</td><td class='num' data-sort='0.679'>0.679</td><td class='num' data-sort='0.7253'>0.725</td><td class='num' data-sort='0.734'>0.734</td><td class='num' data-sort='0.7056'>0.706</td><td class='num' data-sort='0.6678'>0.668</td><td class='num' data-sort='0.5622'>0.562</td></tr><tr><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='num' data-sort='0.6518'>0.652</td><td class='num' data-sort='0.5883'>0.588</td><td class='num' data-sort='0.6445'>0.644</td><td class='num' data-sort='0.6596'>0.660</td><td class='num' data-sort='0.6889'>0.689</td><td class='num' data-sort='0.6777'>0.678</td></tr><tr><td class='method' data-sort='MI'>MI</td><td class='num' data-sort='0.6474'>0.647</td><td class='num' data-sort='0.6455'>0.645</td><td class='num' data-sort='0.6726'>0.673</td><td class='num' data-sort='0.6829'>0.683</td><td class='num' data-sort='0.6494'>0.649</td><td class='num' data-sort='0.5868'>0.587</td></tr><tr><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='num' data-sort='0.646'>0.646</td><td class='num' data-sort='0.6415'>0.641</td><td class='num' data-sort='0.647'>0.647</td><td class='num' data-sort='0.6725'>0.672</td><td class='num' data-sort='0.6563'>0.656</td><td class='num' data-sort='0.6124'>0.612</td></tr><tr><td class='method' data-sort='GainRatio'>GainRatio</td><td class='num' data-sort='0.6441'>0.644</td><td class='num' data-sort='0.6788'>0.679</td><td class='num' data-sort='0.6807'>0.681</td><td class='num' data-sort='0.6703'>0.670</td><td class='num' data-sort='0.641'>0.641</td><td class='num' data-sort='0.5498'>0.550</td></tr><tr><td class='method' data-sort='RFImportance'>RFImportance</td><td class='num' data-sort='0.6432'>0.643</td><td class='num' data-sort='0.6282'>0.628</td><td class='num' data-sort='0.6529'>0.653</td><td class='num' data-sort='0.6762'>0.676</td><td class='num' data-sort='0.6613'>0.661</td><td class='num' data-sort='0.5974'>0.597</td></tr><tr><td class='method' data-sort='mRMR'>mRMR</td><td class='num' data-sort='0.6036'>0.604</td><td class='num' data-sort='0.5276'>0.528</td><td class='num' data-sort='0.5784'>0.578</td><td class='num' data-sort='0.6139'>0.614</td><td class='num' data-sort='0.656'>0.656</td><td class='num' data-sort='0.651'>0.651</td></tr><tr><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='num' data-sort='0.4472'>0.447</td><td class='num' data-sort='0.5094'>0.509</td><td class='num' data-sort='0.5187'>0.519</td><td class='num' data-sort='0.4808'>0.481</td><td class='num' data-sort='0.4188'>0.419</td><td class='num' data-sort='0.3083'>0.308</td></tr><tr><td class='method' data-sort='CART'>CART</td><td class='num' data-sort='0.422'>0.422</td><td class='num' data-sort='0.4838'>0.484</td><td class='num' data-sort='0.4556'>0.456</td><td class='num' data-sort='0.4444'>0.444</td><td class='num' data-sort='0.4018'>0.402</td><td class='num' data-sort='0.3241'>0.324</td></tr><tr><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='num' data-sort='0.413'>0.413</td><td class='num' data-sort='0.4177'>0.418</td><td class='num' data-sort='0.3958'>0.396</td><td class='num' data-sort='0.4132'>0.413</td><td class='num' data-sort='0.4337'>0.434</td><td class='num' data-sort='0.4045'>0.405</td></tr><tr><td class='method' data-sort='Lasso'>Lasso</td><td class='num' data-sort='0.3623'>0.362</td><td class='num' data-sort='0.4208'>0.421</td><td class='num' data-sort='0.4216'>0.422</td><td class='num' data-sort='0.3889'>0.389</td><td class='num' data-sort='0.3343'>0.334</td><td class='num' data-sort='0.2457'>0.246</td></tr><tr><td class='method' data-sort='SFS'>SFS</td><td class='num' data-sort='0.2668'>0.267</td><td class='num' data-sort='0.3677'>0.368</td><td class='num' data-sort='0.2781'>0.278</td><td class='num' data-sort='0.2392'>0.239</td><td class='num' data-sort='0.2248'>0.225</td><td class='num' data-sort='0.1758'>0.176</td></tr><tr><td class='method' data-sort='LOCO'>LOCO</td><td class='num' data-sort='0.1727'>0.173</td><td class='num' data-sort='0.2993'>0.299</td><td class='num' data-sort='0.2199'>0.220</td><td class='num' data-sort='0.1514'>0.151</td><td class='num' data-sort='0.1136'>0.114</td><td class='num' data-sort='0.0795'>0.080</td></tr><tr><td class='method' data-sort='SBE'>SBE</td><td class='num' data-sort='0.1323'>0.132</td><td class='num' data-sort='0.1945'>0.195</td><td class='num' data-sort='0.1753'>0.175</td><td class='num' data-sort='0.1437'>0.144</td><td class='num' data-sort='0.0949'>0.095</td><td class='num' data-sort='0.0594'>0.059</td></tr><tr><td class='method' data-sort='Random'>Random</td><td class='num' data-sort='0.0006'>0.001</td><td class='num' data-sort='-0.0013'>-0.001</td><td class='num' data-sort='0.001'>0.001</td><td class='num' data-sort='-0.0007'>-0.001</td><td class='num' data-sort='0.002'>0.002</td><td class='num' data-sort='0.002'>0.002</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>method,stability,stability_budget_0,stability_budget_1,stability_budget_2,stability_budget_3,stability_budget_4
36
  F-Test,0.679,0.7253,0.734,0.7056,0.6678,0.5622
37
  LaplacianScore,0.6518,0.5883,0.6445,0.6596,0.6889,0.6777
38
  MI,0.6474,0.6455,0.6726,0.6829,0.6494,0.5868
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Medals mark the three best methods.</span><button onclick="saCsv('selectarena_stability.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='' title='Medals for the three highest stability values. Click a method to open its paper.'>Method</th><th class='num' title='Mean over datasets and budgets.'>Stability ↑</th><th class='num'>Budget 1 (smallest)</th><th class='num'>Budget 2</th><th class='num'>Budget 3</th><th class='num'>Budget 4</th><th class='num'>Budget 5 (largest)</th></tr></thead><tbody><tr><td class='method' data-sort='F-Test'><span class='medal' role='img' aria-label='gold medal'>🥇</span><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='num' data-sort='0.679'>0.679</td><td class='num' data-sort='0.7253'>0.725</td><td class='num' data-sort='0.734'>0.734</td><td class='num' data-sort='0.7056'>0.706</td><td class='num' data-sort='0.6678'>0.668</td><td class='num' data-sort='0.5622'>0.562</td></tr><tr><td class='method' data-sort='LaplacianScore'><span class='medal' role='img' aria-label='silver medal'>🥈</span><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='num' data-sort='0.6518'>0.652</td><td class='num' data-sort='0.5883'>0.588</td><td class='num' data-sort='0.6445'>0.644</td><td class='num' data-sort='0.6596'>0.660</td><td class='num' data-sort='0.6889'>0.689</td><td class='num' data-sort='0.6777'>0.678</td></tr><tr><td class='method' data-sort='MI'><span class='medal' role='img' aria-label='bronze medal'>🥉</span><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='num' data-sort='0.6474'>0.647</td><td class='num' data-sort='0.6455'>0.645</td><td class='num' data-sort='0.6726'>0.673</td><td class='num' data-sort='0.6829'>0.683</td><td class='num' data-sort='0.6494'>0.649</td><td class='num' data-sort='0.5868'>0.587</td></tr><tr><td class='method' data-sort='(R)ReliefF'><span class='medal'></span><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='num' data-sort='0.646'>0.646</td><td class='num' data-sort='0.6415'>0.641</td><td class='num' data-sort='0.647'>0.647</td><td class='num' data-sort='0.6725'>0.672</td><td class='num' data-sort='0.6563'>0.656</td><td class='num' data-sort='0.6124'>0.612</td></tr><tr><td class='method' data-sort='GainRatio'><span class='medal'></span><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='num' data-sort='0.6441'>0.644</td><td class='num' data-sort='0.6788'>0.679</td><td class='num' data-sort='0.6807'>0.681</td><td class='num' data-sort='0.6703'>0.670</td><td class='num' data-sort='0.641'>0.641</td><td class='num' data-sort='0.5498'>0.550</td></tr><tr><td class='method' data-sort='RFImportance'><span class='medal'></span><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='num' data-sort='0.6432'>0.643</td><td class='num' data-sort='0.6282'>0.628</td><td class='num' data-sort='0.6529'>0.653</td><td class='num' data-sort='0.6762'>0.676</td><td class='num' data-sort='0.6613'>0.661</td><td class='num' data-sort='0.5974'>0.597</td></tr><tr><td class='method' data-sort='mRMR'><span class='medal'></span><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='num' data-sort='0.6036'>0.604</td><td class='num' data-sort='0.5276'>0.528</td><td class='num' data-sort='0.5784'>0.578</td><td class='num' data-sort='0.6139'>0.614</td><td class='num' data-sort='0.656'>0.656</td><td class='num' data-sort='0.651'>0.651</td></tr><tr><td class='method' data-sort='ElasticNet'><span class='medal'></span><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='num' data-sort='0.4472'>0.447</td><td class='num' data-sort='0.5094'>0.509</td><td class='num' data-sort='0.5187'>0.519</td><td class='num' data-sort='0.4808'>0.481</td><td class='num' data-sort='0.4188'>0.419</td><td class='num' data-sort='0.3083'>0.308</td></tr><tr><td class='method' data-sort='CART'><span class='medal'></span><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='num' data-sort='0.422'>0.422</td><td class='num' data-sort='0.4838'>0.484</td><td class='num' data-sort='0.4556'>0.456</td><td class='num' data-sort='0.4444'>0.444</td><td class='num' data-sort='0.4018'>0.402</td><td class='num' data-sort='0.3241'>0.324</td></tr><tr><td class='method' data-sort='MarkovBlanket'><span class='medal'></span><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='num' data-sort='0.413'>0.413</td><td class='num' data-sort='0.4177'>0.418</td><td class='num' data-sort='0.3958'>0.396</td><td class='num' data-sort='0.4132'>0.413</td><td class='num' data-sort='0.4337'>0.434</td><td class='num' data-sort='0.4045'>0.405</td></tr><tr><td class='method' data-sort='Lasso'><span class='medal'></span><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='num' data-sort='0.3623'>0.362</td><td class='num' data-sort='0.4208'>0.421</td><td class='num' data-sort='0.4216'>0.422</td><td class='num' data-sort='0.3889'>0.389</td><td class='num' data-sort='0.3343'>0.334</td><td class='num' data-sort='0.2457'>0.246</td></tr><tr><td class='method' data-sort='SFS'><span class='medal'></span><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='num' data-sort='0.2668'>0.267</td><td class='num' data-sort='0.3677'>0.368</td><td class='num' data-sort='0.2781'>0.278</td><td class='num' data-sort='0.2392'>0.239</td><td class='num' data-sort='0.2248'>0.225</td><td class='num' data-sort='0.1758'>0.176</td></tr><tr><td class='method' data-sort='LOCO'><span class='medal'></span><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='num' data-sort='0.1727'>0.173</td><td class='num' data-sort='0.2993'>0.299</td><td class='num' data-sort='0.2199'>0.220</td><td class='num' data-sort='0.1514'>0.151</td><td class='num' data-sort='0.1136'>0.114</td><td class='num' data-sort='0.0795'>0.080</td></tr><tr><td class='method' data-sort='SBE'><span class='medal'></span><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='num' data-sort='0.1323'>0.132</td><td class='num' data-sort='0.1945'>0.195</td><td class='num' data-sort='0.1753'>0.175</td><td class='num' data-sort='0.1437'>0.144</td><td class='num' data-sort='0.0949'>0.095</td><td class='num' data-sort='0.0594'>0.059</td></tr><tr><td class='method' data-sort='Random'><span class='medal'></span>Random</td><td class='num' data-sort='0.0006'>0.001</td><td class='num' data-sort='-0.0013'>-0.001</td><td class='num' data-sort='0.001'>0.001</td><td class='num' data-sort='-0.0007'>-0.001</td><td class='num' data-sort='0.002'>0.002</td><td class='num' data-sort='0.002'>0.002</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>method,stability,stability_budget_0,stability_budget_1,stability_budget_2,stability_budget_3,stability_budget_4
38
  F-Test,0.679,0.7253,0.734,0.7056,0.6678,0.5622
39
  LaplacianScore,0.6518,0.5883,0.6445,0.6596,0.6889,0.6777
40
  MI,0.6474,0.6455,0.6726,0.6829,0.6494,0.5868
data/subsets/tasks_all/models_all/budgets_0/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1343.0'>1343<span class='sub'>+52 / −46</span></td><td class='num' data-sort='29.9'>29.9</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1315.0'>1315<span class='sub'>+47 / −42</span></td><td class='num' data-sort='27.6'>27.6</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1308.0'>1308<span class='sub'>+44 / −41</span></td><td class='num' data-sort='26.8'>26.8</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1290.0'>1290<span class='sub'>+41 / −40</span></td><td class='num' data-sort='28.3'>28.3</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1290.0'>1290<span class='sub'>+34 / −35</span></td><td class='num' data-sort='28.7'>28.7</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1284.0'>1284<span class='sub'>+42 / −40</span></td><td class='num' data-sort='25.6'>25.6</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='7.5'>7.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1270.0'>1270<span class='sub'>+44 / −46</span></td><td class='num' data-sort='23.7'>23.7</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='941.9'>941.9</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1255.0'>1255<span class='sub'>+39 / −38</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1250.0'>1250<span class='sub'>+36 / −41</span></td><td class='num' data-sort='24.1'>24.1</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='2.2'>2.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1242.0'>1242<span class='sub'>+37 / −39</span></td><td class='num' data-sort='23.6'>23.6</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1206.0'>1206<span class='sub'>+44 / −45</span></td><td class='num' data-sort='20.4'>20.4</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='14.2'>14.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+48 / −47</span></td><td class='num' data-sort='12.5'>12.5</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3579.1'>3,579.1</td><td class='num' data-sort='49.6'>49.6</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1176.0'>1176<span class='sub'>+42 / −41</span></td><td class='num' data-sort='15.3'>15.3</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+46 / −54</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='963.0'>963<span class='sub'>+61 / −72</span></td><td class='num' data-sort='-14.7'>-14.7</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1343.0,1297.0,1395.0,29.9,0.628,0.805,0.2,0.0
37
  2,CART,Non-search,Tree,1315.0,1273.0,1362.0,27.6,0.484,0.731,0.1,0.0
38
  3,MarkovBlanket,Search,Backward search,1308.0,1267.0,1352.0,26.8,0.418,0.465,0.2,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1343.0'>1343<span class='sub'>+52 / −46</span></td><td class='num' data-sort='29.9'>29.9</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1315.0'>1315<span class='sub'>+47 / −42</span></td><td class='num' data-sort='27.6'>27.6</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1308.0'>1308<span class='sub'>+44 / −41</span></td><td class='num' data-sort='26.8'>26.8</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1290.0'>1290<span class='sub'>+41 / −40</span></td><td class='num' data-sort='28.3'>28.3</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1290.0'>1290<span class='sub'>+34 / −35</span></td><td class='num' data-sort='28.7'>28.7</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1284.0'>1284<span class='sub'>+42 / −40</span></td><td class='num' data-sort='25.6'>25.6</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='7.5'>7.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1270.0'>1270<span class='sub'>+44 / −46</span></td><td class='num' data-sort='23.7'>23.7</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='941.9'>941.9</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1255.0'>1255<span class='sub'>+39 / −38</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1250.0'>1250<span class='sub'>+36 / −41</span></td><td class='num' data-sort='24.1'>24.1</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='2.2'>2.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1242.0'>1242<span class='sub'>+37 / −39</span></td><td class='num' data-sort='23.6'>23.6</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1206.0'>1206<span class='sub'>+44 / −45</span></td><td class='num' data-sort='20.4'>20.4</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='14.2'>14.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+48 / −47</span></td><td class='num' data-sort='12.5'>12.5</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3579.1'>3,579.1</td><td class='num' data-sort='49.6'>49.6</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1176.0'>1176<span class='sub'>+42 / −41</span></td><td class='num' data-sort='15.3'>15.3</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+46 / −54</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='963.0'>963<span class='sub'>+61 / −72</span></td><td class='num' data-sort='-14.7'>-14.7</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1343.0,1297.0,1395.0,29.9,0.628,0.805,0.2,0.0
39
  2,CART,Non-search,Tree,1315.0,1273.0,1362.0,27.6,0.484,0.731,0.1,0.0
40
  3,MarkovBlanket,Search,Backward search,1308.0,1267.0,1352.0,26.8,0.418,0.465,0.2,0.0
data/subsets/tasks_all/models_all/budgets_0/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='96.7' x2='96.7' y1='16.2' y2='368.0'/><text class='sa-text' x='96.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='206.1' x2='206.1' y1='16.2' y2='368.0'/><text class='sa-text' x='206.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='315.6' x2='315.6' y1='16.2' y2='368.0'/><text class='sa-text' x='315.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='425.1' x2='425.1' y1='16.2' y2='368.0'/><text class='sa-text' x='425.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='534.5' x2='534.5' y1='16.2' y2='368.0'/><text class='sa-text' x='534.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='644.0' x2='644.0' y1='16.2' y2='368.0'/><text class='sa-text' x='644.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='753.5' x2='753.5' y1='16.2' y2='368.0'/><text class='sa-text' x='753.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='298.6' y2='298.6'/><text class='sa-text' x='58.0' y='304.6' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='245.2' y2='245.2'/><text class='sa-text' x='58.0' y='251.2' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='191.8' y2='191.8'/><text class='sa-text' x='58.0' y='197.8' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='138.4' y2='138.4'/><text class='sa-text' x='58.0' y='144.4' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='85.0' y2='85.0'/><text class='sa-text' x='58.0' y='91.0' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='31.6' y2='31.6'/><text class='sa-text' x='58.0' y='37.6' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 305.1,49.9 496.6,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='496.6' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='502.9' cy='45.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='305.1' cy='49.9' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='446.4' cy='57.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='461.7' cy='65.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.3' cy='78.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='454.0' cy='79.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='684.5' cy='94.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='778.9' cy='98.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='442.0' cy='100.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.9' cy='133.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='731.5' cy='188.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='218.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='496.6' y1='32.2' x2='589.1' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='589.1' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='502.9' y1='45.6' x2='590.4' y2='57.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='590.4' y='57.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='305.1' y1='49.9' x2='265.5' y2='63.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='265.5' y='63.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='446.4' y1='57.0' x2='412.4' y2='45.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='412.4' y='45.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='461.7' y1='65.6' x2='572.0' y2='86.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='572.0' y='86.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='686.3' y1='78.3' x2='794.5' y2='58.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='794.5' y='58.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='454.0' y1='79.6' x2='552.2' y2='117.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='552.2' y='117.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='684.5' y1='94.4' x2='723.4' y2='120.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='723.4' y='120.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='778.9' y1='98.7' x2='806.0' y2='86.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='806.0' y='86.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='442.0' y1='100.3' x2='413.7' y2='121.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='413.7' y='121.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='797.0' y='149.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='731.5' y1='188.2' x2='698.7' y2='201.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='698.7' y='201.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='812.3' y1='218.4' x2='787.8' y2='232.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='787.8' y='232.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='496.6' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.9%&#10;Selection time per 1k features: 45 s&#10;Datasets: 71&#10;Next to MI: 28.7%, 51.4 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.9%&#10;Selection time per 1k features: 45 s&#10;Datasets: 71&#10;Next to MI: 28.7%, 51.4 s'/><circle class='sa-hit' cx='502.9' cy='45.6' r='10.5' data-tip='MI&#10;Improvement over Random: 28.7%&#10;Selection time per 1k features: 51.4 s&#10;Datasets: 71&#10;Next to RFImportance: 29.9%, 45 s' aria-label='MI&#10;Improvement over Random: 28.7%&#10;Selection time per 1k features: 51.4 s&#10;Datasets: 71&#10;Next to RFImportance: 29.9%, 45 s'/><circle class='sa-hit' cx='305.1' cy='49.9' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 28.3%&#10;Selection time per 1k features: 0.802 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 28.3%&#10;Selection time per 1k features: 0.802 s&#10;Datasets: 71'/><circle class='sa-hit' cx='446.4' cy='57.0' r='10.5' data-tip='CART&#10;Improvement over Random: 27.6%&#10;Selection time per 1k features: 15.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 26.8%, 21.6 s' aria-label='CART&#10;Improvement over Random: 27.6%&#10;Selection time per 1k features: 15.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 26.8%, 21.6 s'/><circle class='sa-hit' cx='461.7' cy='65.6' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 26.8%&#10;Selection time per 1k features: 21.6 s&#10;Datasets: 71&#10;Next to CART: 27.6%, 15.7 s&#10;Next to mRMR: 25.5%, 18.4 s' aria-label='MarkovBlanket&#10;Improvement over Random: 26.8%&#10;Selection time per 1k features: 21.6 s&#10;Datasets: 71&#10;Next to CART: 27.6%, 15.7 s&#10;Next to mRMR: 25.5%, 18.4 s'/><circle class='sa-hit' cx='686.3' cy='78.3' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 25.6%&#10;Selection time per 1k features: 2.43e+03 s&#10;Datasets: 71&#10;Next to Lasso: 24.1%, 2.35e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 25.6%&#10;Selection time per 1k features: 2.43e+03 s&#10;Datasets: 71&#10;Next to Lasso: 24.1%, 2.35e+03 s'/><circle class='sa-hit' cx='454.0' cy='79.6' r='10.5' data-tip='mRMR&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 18.4 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 26.8%, 21.6 s' aria-label='mRMR&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 18.4 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 26.8%, 21.6 s'/><circle class='sa-hit' cx='684.5' cy='94.4' r='10.5' data-tip='Lasso&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 2.35e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.6%, 2.43e+03 s' aria-label='Lasso&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 2.35e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.6%, 2.43e+03 s'/><circle class='sa-hit' cx='778.9' cy='98.7' r='10.5' data-tip='SFS&#10;Improvement over Random: 23.7%&#10;Selection time per 1k features: 1.71e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 23.7%&#10;Selection time per 1k features: 1.71e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='442.0' cy='100.3' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 23.6%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 23.6%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.9' cy='133.8' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 20.4%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 20.4%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='731.5' cy='188.2' r='10.5' data-tip='LOCO&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 6.3e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 6.3e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='218.4' r='10.5' data-tip='SBE&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 3.45e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 3.45e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0113 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0113 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,29.9487,44.9997,71,True
37
  MI,28.6934,51.3992,71,False
38
  F-Test,28.2875,0.8021,71,True
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='96.7' x2='96.7' y1='16.2' y2='368.0'/><text class='sa-text' x='96.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='206.1' x2='206.1' y1='16.2' y2='368.0'/><text class='sa-text' x='206.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='315.6' x2='315.6' y1='16.2' y2='368.0'/><text class='sa-text' x='315.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='425.1' x2='425.1' y1='16.2' y2='368.0'/><text class='sa-text' x='425.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='534.5' x2='534.5' y1='16.2' y2='368.0'/><text class='sa-text' x='534.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='644.0' x2='644.0' y1='16.2' y2='368.0'/><text class='sa-text' x='644.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='753.5' x2='753.5' y1='16.2' y2='368.0'/><text class='sa-text' x='753.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='298.6' y2='298.6'/><text class='sa-text' x='58.0' y='304.6' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='245.2' y2='245.2'/><text class='sa-text' x='58.0' y='251.2' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='191.8' y2='191.8'/><text class='sa-text' x='58.0' y='197.8' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='138.4' y2='138.4'/><text class='sa-text' x='58.0' y='144.4' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='85.0' y2='85.0'/><text class='sa-text' x='58.0' y='91.0' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='31.6' y2='31.6'/><text class='sa-text' x='58.0' y='37.6' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 305.1,49.9 496.6,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='496.6' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='502.9' cy='45.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='305.1' cy='49.9' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='446.4' cy='57.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='461.7' cy='65.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.3' cy='78.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='454.0' cy='79.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='684.5' cy='94.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='778.9' cy='98.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='442.0' cy='100.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.9' cy='133.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='731.5' cy='188.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='218.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='496.6' y1='32.2' x2='587.9' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='587.9' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='502.9' y1='45.6' x2='582.9' y2='53.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='582.9' y='53.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='305.1' y1='49.9' x2='265.5' y2='63.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='265.5' y='63.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='446.4' y1='57.0' x2='416.8' y2='42.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='416.8' y='42.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='461.7' y1='65.6' x2='586.5' y2='93.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='586.5' y='93.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='686.3' y1='78.3' x2='794.5' y2='52.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='794.5' y='52.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='454.0' y1='79.6' x2='556.0' y2='123.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='556.0' y='123.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='684.5' y1='94.4' x2='729.1' y2='121.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='729.1' y='121.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='778.9' y1='98.7' x2='802.2' y2='89.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='802.2' y='89.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='442.0' y1='100.3' x2='408.7' y2='124.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='408.7' y='124.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='795.9' y1='133.8' x2='797.0' y2='155.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='797.0' y='155.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='731.5' y1='188.2' x2='698.7' y2='201.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='698.7' y='201.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='812.3' y1='218.4' x2='787.8' y2='232.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='787.8' y='232.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='496.6' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.9%&#10;Selection time per 1k features: 45 s&#10;Datasets: 71&#10;Next to MI: 28.7%, 51.4 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.9%&#10;Selection time per 1k features: 45 s&#10;Datasets: 71&#10;Next to MI: 28.7%, 51.4 s'/><circle class='sa-hit' cx='502.9' cy='45.6' r='10.5' data-tip='MI&#10;Improvement over Random: 28.7%&#10;Selection time per 1k features: 51.4 s&#10;Datasets: 71&#10;Next to RFImportance: 29.9%, 45 s' aria-label='MI&#10;Improvement over Random: 28.7%&#10;Selection time per 1k features: 51.4 s&#10;Datasets: 71&#10;Next to RFImportance: 29.9%, 45 s'/><circle class='sa-hit' cx='305.1' cy='49.9' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 28.3%&#10;Selection time per 1k features: 0.802 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 28.3%&#10;Selection time per 1k features: 0.802 s&#10;Datasets: 71'/><circle class='sa-hit' cx='446.4' cy='57.0' r='10.5' data-tip='CART&#10;Improvement over Random: 27.6%&#10;Selection time per 1k features: 15.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 26.8%, 21.6 s' aria-label='CART&#10;Improvement over Random: 27.6%&#10;Selection time per 1k features: 15.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 26.8%, 21.6 s'/><circle class='sa-hit' cx='461.7' cy='65.6' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 26.8%&#10;Selection time per 1k features: 21.6 s&#10;Datasets: 71&#10;Next to CART: 27.6%, 15.7 s&#10;Next to mRMR: 25.5%, 18.4 s' aria-label='MarkovBlanket&#10;Improvement over Random: 26.8%&#10;Selection time per 1k features: 21.6 s&#10;Datasets: 71&#10;Next to CART: 27.6%, 15.7 s&#10;Next to mRMR: 25.5%, 18.4 s'/><circle class='sa-hit' cx='686.3' cy='78.3' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 25.6%&#10;Selection time per 1k features: 2.43e+03 s&#10;Datasets: 71&#10;Next to Lasso: 24.1%, 2.35e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 25.6%&#10;Selection time per 1k features: 2.43e+03 s&#10;Datasets: 71&#10;Next to Lasso: 24.1%, 2.35e+03 s'/><circle class='sa-hit' cx='454.0' cy='79.6' r='10.5' data-tip='mRMR&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 18.4 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 26.8%, 21.6 s' aria-label='mRMR&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 18.4 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 26.8%, 21.6 s'/><circle class='sa-hit' cx='684.5' cy='94.4' r='10.5' data-tip='Lasso&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 2.35e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.6%, 2.43e+03 s' aria-label='Lasso&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 2.35e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.6%, 2.43e+03 s'/><circle class='sa-hit' cx='778.9' cy='98.7' r='10.5' data-tip='SFS&#10;Improvement over Random: 23.7%&#10;Selection time per 1k features: 1.71e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 23.7%&#10;Selection time per 1k features: 1.71e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='442.0' cy='100.3' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 23.6%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 23.6%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.9' cy='133.8' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 20.4%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 20.4%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='731.5' cy='188.2' r='10.5' data-tip='LOCO&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 6.3e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 6.3e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='218.4' r='10.5' data-tip='SBE&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 3.45e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 3.45e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0113 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0113 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,29.9487,44.9997,71,True
37
  MI,28.6934,51.3992,71,False
38
  F-Test,28.2875,0.8021,71,True
data/subsets/tasks_all/models_all/budgets_0/per_model_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
data/subsets/tasks_all/models_all/budgets_1/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+45 / −44</span></td><td class='num' data-sort='29.3'>29.3</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1346.0'>1346<span class='sub'>+42 / −43</span></td><td class='num' data-sort='26.0'>26.0</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1335.0'>1335<span class='sub'>+40 / −39</span></td><td class='num' data-sort='27.1'>27.1</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1323.0'>1323<span class='sub'>+44 / −43</span></td><td class='num' data-sort='25.4'>25.4</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.3'>2.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1316.0'>1316<span class='sub'>+34 / −30</span></td><td class='num' data-sort='27.5'>27.5</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1307.0'>1307<span class='sub'>+36 / −33</span></td><td class='num' data-sort='24.6'>24.6</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1298.0'>1298<span class='sub'>+36 / −40</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+34 / −36</span></td><td class='num' data-sort='23.7'>23.7</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1268.0'>1268<span class='sub'>+39 / −40</span></td><td class='num' data-sort='24.6'>24.6</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1257.0'>1257<span class='sub'>+40 / −39</span></td><td class='num' data-sort='21.3'>21.3</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1256.0'>1256<span class='sub'>+43 / −44</span></td><td class='num' data-sort='21.6'>21.6</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1634.9'>1,634.9</td><td class='num' data-sort='43.8'>43.8</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1227.0'>1227<span class='sub'>+49 / −50</span></td><td class='num' data-sort='14.7'>14.7</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3166.2'>3,166.2</td><td class='num' data-sort='48.9'>48.9</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1219.0'>1219<span class='sub'>+43 / −42</span></td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='237.3'>237.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1025.0'>1025<span class='sub'>+52 / −64</span></td><td class='num' data-sort='-22.8'>-22.8</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+41 / −47</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1366.0,1322.0,1411.0,29.3,0.653,0.741,0.2,0.0
37
  2,ElasticNet,Non-search,Regularization,1346.0,1303.0,1388.0,26.0,0.519,0.873,7.6,0.0
38
  3,CART,Non-search,Tree,1335.0,1296.0,1375.0,27.1,0.456,0.638,0.1,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+45 / −44</span></td><td class='num' data-sort='29.3'>29.3</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1346.0'>1346<span class='sub'>+42 / −43</span></td><td class='num' data-sort='26.0'>26.0</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1335.0'>1335<span class='sub'>+40 / −39</span></td><td class='num' data-sort='27.1'>27.1</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1323.0'>1323<span class='sub'>+44 / −43</span></td><td class='num' data-sort='25.4'>25.4</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.3'>2.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1316.0'>1316<span class='sub'>+34 / −30</span></td><td class='num' data-sort='27.5'>27.5</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1307.0'>1307<span class='sub'>+36 / −33</span></td><td class='num' data-sort='24.6'>24.6</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1298.0'>1298<span class='sub'>+36 / −40</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+34 / −36</span></td><td class='num' data-sort='23.7'>23.7</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1268.0'>1268<span class='sub'>+39 / −40</span></td><td class='num' data-sort='24.6'>24.6</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1257.0'>1257<span class='sub'>+40 / −39</span></td><td class='num' data-sort='21.3'>21.3</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1256.0'>1256<span class='sub'>+43 / −44</span></td><td class='num' data-sort='21.6'>21.6</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1634.9'>1,634.9</td><td class='num' data-sort='43.8'>43.8</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1227.0'>1227<span class='sub'>+49 / −50</span></td><td class='num' data-sort='14.7'>14.7</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3166.2'>3,166.2</td><td class='num' data-sort='48.9'>48.9</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1219.0'>1219<span class='sub'>+43 / −42</span></td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='237.3'>237.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1025.0'>1025<span class='sub'>+52 / −64</span></td><td class='num' data-sort='-22.8'>-22.8</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+41 / −47</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1366.0,1322.0,1411.0,29.3,0.653,0.741,0.2,0.0
39
  2,ElasticNet,Non-search,Regularization,1346.0,1303.0,1388.0,26.0,0.519,0.873,7.6,0.0
40
  3,CART,Non-search,Tree,1335.0,1296.0,1375.0,27.1,0.456,0.638,0.1,0.0
data/subsets/tasks_all/models_all/budgets_1/per_model_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
data/subsets/tasks_all/models_all/budgets_2/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+45 / −39</span></td><td class='num' data-sort='24.3'>24.3</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1278.0'>1278<span class='sub'>+42 / −37</span></td><td class='num' data-sort='20.5'>20.5</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+42 / −43</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1272.0'>1272<span class='sub'>+40 / −37</span></td><td class='num' data-sort='20.9'>20.9</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1249.0'>1249<span class='sub'>+40 / −45</span></td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1237.0'>1237<span class='sub'>+32 / −34</span></td><td class='num' data-sort='18.0'>18.0</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1223.0'>1223<span class='sub'>+36 / −35</span></td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1210.0'>1210<span class='sub'>+39 / −38</span></td><td class='num' data-sort='18.5'>18.5</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='14.4'>14.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1194.0'>1194<span class='sub'>+38 / −37</span></td><td class='num' data-sort='16.2'>16.2</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+45 / −46</span></td><td class='num' data-sort='10.4'>10.4</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2621.7'>2,621.7</td><td class='num' data-sort='47.8'>47.8</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1181.0'>1181<span class='sub'>+35 / −37</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1175.0'>1175<span class='sub'>+37 / −42</span></td><td class='num' data-sort='13.6'>13.6</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2398.9'>2,398.9</td><td class='num' data-sort='47.2'>47.2</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1157.0'>1157<span class='sub'>+40 / −41</span></td><td class='num' data-sort='9.3'>9.3</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='241.3'>241.3</td><td class='num' data-sort='6.8'>6.8</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+42 / −49</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='930.0'>930<span class='sub'>+54 / −71</span></td><td class='num' data-sort='-58.2'>-58.2</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1325.0,1286.0,1370.0,24.3,0.676,0.691,0.2,0.0
37
  2,CART,Non-search,Tree,1278.0,1241.0,1320.0,20.5,0.444,0.572,0.1,0.0
38
  3,ElasticNet,Non-search,Regularization,1273.0,1230.0,1315.0,19.8,0.481,0.777,7.6,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+45 / −39</span></td><td class='num' data-sort='24.3'>24.3</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1278.0'>1278<span class='sub'>+42 / −37</span></td><td class='num' data-sort='20.5'>20.5</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+42 / −43</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1272.0'>1272<span class='sub'>+40 / −37</span></td><td class='num' data-sort='20.9'>20.9</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1249.0'>1249<span class='sub'>+40 / −45</span></td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1237.0'>1237<span class='sub'>+32 / −34</span></td><td class='num' data-sort='18.0'>18.0</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1223.0'>1223<span class='sub'>+36 / −35</span></td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1210.0'>1210<span class='sub'>+39 / −38</span></td><td class='num' data-sort='18.5'>18.5</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='14.4'>14.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1194.0'>1194<span class='sub'>+38 / −37</span></td><td class='num' data-sort='16.2'>16.2</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+45 / −46</span></td><td class='num' data-sort='10.4'>10.4</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2621.7'>2,621.7</td><td class='num' data-sort='47.8'>47.8</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1181.0'>1181<span class='sub'>+35 / −37</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1175.0'>1175<span class='sub'>+37 / −42</span></td><td class='num' data-sort='13.6'>13.6</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2398.9'>2,398.9</td><td class='num' data-sort='47.2'>47.2</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1157.0'>1157<span class='sub'>+40 / −41</span></td><td class='num' data-sort='9.3'>9.3</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='241.3'>241.3</td><td class='num' data-sort='6.8'>6.8</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+42 / −49</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='930.0'>930<span class='sub'>+54 / −71</span></td><td class='num' data-sort='-58.2'>-58.2</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1325.0,1286.0,1370.0,24.3,0.676,0.691,0.2,0.0
39
  2,CART,Non-search,Tree,1278.0,1241.0,1320.0,20.5,0.444,0.572,0.1,0.0
40
  3,ElasticNet,Non-search,Regularization,1273.0,1230.0,1315.0,19.8,0.481,0.777,7.6,0.0
data/subsets/tasks_all/models_all/budgets_2/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='135.7' x2='135.7' y1='16.2' y2='368.0'/><text class='sa-text' x='135.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='240.2' x2='240.2' y1='16.2' y2='368.0'/><text class='sa-text' x='240.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='344.6' x2='344.6' y1='16.2' y2='368.0'/><text class='sa-text' x='344.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='449.1' x2='449.1' y1='16.2' y2='368.0'/><text class='sa-text' x='449.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='553.5' x2='553.5' y1='16.2' y2='368.0'/><text class='sa-text' x='553.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='658.0' x2='658.0' y1='16.2' y2='368.0'/><text class='sa-text' x='658.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='762.4' x2='762.4' y1='16.2' y2='368.0'/><text class='sa-text' x='762.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='286.2' y2='286.2'/><text class='sa-text' x='57.9' y='292.2' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='220.4' y2='220.4'/><text class='sa-text' x='57.9' y='226.4' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='154.6' y2='154.6'/><text class='sa-text' x='57.9' y='160.6' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='88.8' y2='88.8'/><text class='sa-text' x='57.9' y='94.8' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='23.0' y2='23.0'/><text class='sa-text' x='57.9' y='29.0' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 333.9,111.7 469.8,82.6 517.4,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='517.4' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='523.2' cy='77.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='469.8' cy='82.6' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='697.8' cy='91.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='803.0' cy='108.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='697.7' cy='111.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='333.9' cy='111.7' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='483.8' cy='114.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='510.3' cy='118.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='465.2' cy='138.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='173.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='810.9' cy='215.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='741.9' cy='229.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='517.4' y1='32.2' x2='641.4' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='641.4' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-dim sa-halo' x='541.4' y='73.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><text class='sa-text sa-halo' x='477.3' y='63.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='697.8' y1='91.9' x2='692.1' y2='70.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='692.1' y='70.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='803.0' y1='108.3' x2='758.5' y2='98.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='758.5' y='98.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='697.7' y1='111.1' x2='721.6' y2='128.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='721.6' y='128.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='333.9' y1='111.7' x2='378.6' y2='130.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='378.6' y='130.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='483.8' y1='114.6' x2='374.9' y2='94.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='374.9' y='94.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='510.3' y1='118.8' x2='549.9' y2='104.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='549.9' y='104.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='465.2' y1='138.7' x2='536.3' y2='152.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='536.3' y='152.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='812.3' y1='173.5' x2='828.6' y2='159.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='159.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='810.9' y1='215.6' x2='827.3' y2='229.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='827.3' y='229.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='741.9' y1='229.4' x2='709.1' y2='243.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='709.1' y='243.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='517.4' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 24.3%&#10;Selection time per 1k features: 45.1 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 24.3%&#10;Selection time per 1k features: 45.1 s&#10;Datasets: 71'/><circle class='sa-hit' cx='523.2' cy='77.1' r='10.5' data-tip='MI&#10;Improvement over Random: 20.9%&#10;Selection time per 1k features: 51.2 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 20.9%&#10;Selection time per 1k features: 51.2 s&#10;Datasets: 71'/><circle class='sa-hit' cx='469.8' cy='82.6' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 20.5%&#10;Selection time per 1k features: 15.8 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 20.5%&#10;Selection time per 1k features: 15.8 s&#10;Datasets: 71'/><circle class='sa-hit' cx='697.8' cy='91.9' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 2.4e+03 s&#10;Datasets: 71&#10;Next to Lasso: 18.3%, 2.4e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 2.4e+03 s&#10;Datasets: 71&#10;Next to Lasso: 18.3%, 2.4e+03 s'/><circle class='sa-hit' cx='803.0' cy='108.3' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 18.5%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 18.5%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='697.7' cy='111.1' r='10.5' data-tip='Lasso&#10;Improvement over Random: 18.3%&#10;Selection time per 1k features: 2.4e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 19.8%, 2.4e+03 s' aria-label='Lasso&#10;Improvement over Random: 18.3%&#10;Selection time per 1k features: 2.4e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 19.8%, 2.4e+03 s'/><circle class='sa-hit' cx='333.9' cy='111.7' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 18.3%&#10;Selection time per 1k features: 0.789 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 18.3%&#10;Selection time per 1k features: 0.789 s&#10;Datasets: 71'/><circle class='sa-hit' cx='483.8' cy='114.6' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 21.5 s&#10;Datasets: 71' aria-label='MarkovBlanket&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 21.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='510.3' cy='118.8' r='10.5' data-tip='mRMR&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='465.2' cy='138.7' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='173.5' r='10.5' data-tip='SFS&#10;Improvement over Random: 13.6%&#10;Selection time per 1k features: 3e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 13.6%&#10;Selection time per 1k features: 3e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='810.9' cy='215.6' r='10.5' data-tip='SBE&#10;Improvement over Random: 10.4%&#10;Selection time per 1k features: 2.91e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 10.4%&#10;Selection time per 1k features: 2.91e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='741.9' cy='229.4' r='10.5' data-tip='LOCO&#10;Improvement over Random: 9.3%&#10;Selection time per 1k features: 6.35e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 9.3%&#10;Selection time per 1k features: 6.35e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0048 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0048 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,24.3008,45.0766,71,True
37
  MI,20.8893,51.2055,71,False
38
  CART,20.4694,15.7754,71,True
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='135.7' x2='135.7' y1='16.2' y2='368.0'/><text class='sa-text' x='135.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='240.2' x2='240.2' y1='16.2' y2='368.0'/><text class='sa-text' x='240.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='344.6' x2='344.6' y1='16.2' y2='368.0'/><text class='sa-text' x='344.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='449.1' x2='449.1' y1='16.2' y2='368.0'/><text class='sa-text' x='449.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='553.5' x2='553.5' y1='16.2' y2='368.0'/><text class='sa-text' x='553.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='658.0' x2='658.0' y1='16.2' y2='368.0'/><text class='sa-text' x='658.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='762.4' x2='762.4' y1='16.2' y2='368.0'/><text class='sa-text' x='762.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='286.2' y2='286.2'/><text class='sa-text' x='57.9' y='292.2' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='220.4' y2='220.4'/><text class='sa-text' x='57.9' y='226.4' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='154.6' y2='154.6'/><text class='sa-text' x='57.9' y='160.6' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='88.8' y2='88.8'/><text class='sa-text' x='57.9' y='94.8' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='23.0' y2='23.0'/><text class='sa-text' x='57.9' y='29.0' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 333.9,111.7 469.8,82.6 517.4,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='517.4' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='523.2' cy='77.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='469.8' cy='82.6' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='697.8' cy='91.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='803.0' cy='108.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='697.7' cy='111.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='333.9' cy='111.7' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='483.8' cy='114.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='510.3' cy='118.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='465.2' cy='138.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='173.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='810.9' cy='215.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='741.9' cy='229.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='517.4' y1='32.2' x2='641.4' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='641.4' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-dim sa-halo' x='541.4' y='73.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><text class='sa-text sa-halo' x='477.3' y='63.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='697.8' y1='91.9' x2='688.3' y2='66.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='688.3' y='66.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='803.0' y1='108.3' x2='762.3' y2='98.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='762.3' y='98.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='697.7' y1='111.1' x2='721.6' y2='131.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='721.6' y='131.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='333.9' y1='111.7' x2='376.7' y2='128.9' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='376.7' y='128.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='483.8' y1='114.6' x2='368.0' y2='96.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='368.0' y='96.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='510.3' y1='118.8' x2='549.9' y2='104.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='549.9' y='104.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='465.2' y1='138.7' x2='536.3' y2='152.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='536.3' y='152.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='812.3' y1='173.5' x2='828.6' y2='159.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='159.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='810.9' y1='215.6' x2='827.3' y2='229.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='827.3' y='229.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='741.9' y1='229.4' x2='709.1' y2='243.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='709.1' y='243.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='517.4' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 24.3%&#10;Selection time per 1k features: 45.1 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 24.3%&#10;Selection time per 1k features: 45.1 s&#10;Datasets: 71'/><circle class='sa-hit' cx='523.2' cy='77.1' r='10.5' data-tip='MI&#10;Improvement over Random: 20.9%&#10;Selection time per 1k features: 51.2 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 20.9%&#10;Selection time per 1k features: 51.2 s&#10;Datasets: 71'/><circle class='sa-hit' cx='469.8' cy='82.6' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 20.5%&#10;Selection time per 1k features: 15.8 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 20.5%&#10;Selection time per 1k features: 15.8 s&#10;Datasets: 71'/><circle class='sa-hit' cx='697.8' cy='91.9' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 2.4e+03 s&#10;Datasets: 71&#10;Next to Lasso: 18.3%, 2.4e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 2.4e+03 s&#10;Datasets: 71&#10;Next to Lasso: 18.3%, 2.4e+03 s'/><circle class='sa-hit' cx='803.0' cy='108.3' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 18.5%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 18.5%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='697.7' cy='111.1' r='10.5' data-tip='Lasso&#10;Improvement over Random: 18.3%&#10;Selection time per 1k features: 2.4e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 19.8%, 2.4e+03 s' aria-label='Lasso&#10;Improvement over Random: 18.3%&#10;Selection time per 1k features: 2.4e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 19.8%, 2.4e+03 s'/><circle class='sa-hit' cx='333.9' cy='111.7' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 18.3%&#10;Selection time per 1k features: 0.789 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 18.3%&#10;Selection time per 1k features: 0.789 s&#10;Datasets: 71'/><circle class='sa-hit' cx='483.8' cy='114.6' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 21.5 s&#10;Datasets: 71' aria-label='MarkovBlanket&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 21.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='510.3' cy='118.8' r='10.5' data-tip='mRMR&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='465.2' cy='138.7' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='173.5' r='10.5' data-tip='SFS&#10;Improvement over Random: 13.6%&#10;Selection time per 1k features: 3e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 13.6%&#10;Selection time per 1k features: 3e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='810.9' cy='215.6' r='10.5' data-tip='SBE&#10;Improvement over Random: 10.4%&#10;Selection time per 1k features: 2.91e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 10.4%&#10;Selection time per 1k features: 2.91e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='741.9' cy='229.4' r='10.5' data-tip='LOCO&#10;Improvement over Random: 9.3%&#10;Selection time per 1k features: 6.35e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 9.3%&#10;Selection time per 1k features: 6.35e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0048 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0048 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,24.3008,45.0766,71,True
37
  MI,20.8893,51.2055,71,False
38
  CART,20.4694,15.7754,71,True
data/subsets/tasks_all/models_all/budgets_2/per_model_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
data/subsets/tasks_all/models_all/budgets_3/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1288.0'>1288<span class='sub'>+43 / −40</span></td><td class='num' data-sort='18.0'>18.0</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1232.0'>1232<span class='sub'>+40 / −31</span></td><td class='num' data-sort='14.4'>14.4</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1218.0'>1218<span class='sub'>+38 / −35</span></td><td class='num' data-sort='14.2'>14.2</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1212.0'>1212<span class='sub'>+39 / −41</span></td><td class='num' data-sort='13.5'>13.5</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+38 / −38</span></td><td class='num' data-sort='12.6'>12.6</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+33 / −32</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+38 / −35</span></td><td class='num' data-sort='12.9'>12.9</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1158.0'>1158<span class='sub'>+31 / −36</span></td><td class='num' data-sort='11.3'>11.3</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1155.0'>1155<span class='sub'>+44 / −49</span></td><td class='num' data-sort='6.3'>6.3</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1890.2'>1,890.2</td><td class='num' data-sort='44.9'>44.9</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1155.0'>1155<span class='sub'>+35 / −36</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1134.0'>1134<span class='sub'>+34 / −34</span></td><td class='num' data-sort='11.6'>11.6</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1127.0'>1127<span class='sub'>+38 / −36</span></td><td class='num' data-sort='5.0'>5.0</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='246.0'>246.0</td><td class='num' data-sort='12.3'>12.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1124.0'>1124<span class='sub'>+33 / −36</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2771.3'>2,771.3</td><td class='num' data-sort='48.7'>48.7</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+40 / −46</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='929.0'>929<span class='sub'>+52 / −60</span></td><td class='num' data-sort='-89.9'>-89.9</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1288.0,1248.0,1331.0,18.0,0.661,0.6,0.2,0.0
37
  2,CART,Non-search,Tree,1232.0,1201.0,1272.0,14.4,0.402,0.507,0.1,0.0
38
  3,MI,Non-search,Information theory,1218.0,1183.0,1256.0,14.2,0.649,0.723,0.4,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1288.0'>1288<span class='sub'>+43 / −40</span></td><td class='num' data-sort='18.0'>18.0</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1232.0'>1232<span class='sub'>+40 / −31</span></td><td class='num' data-sort='14.4'>14.4</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1218.0'>1218<span class='sub'>+38 / −35</span></td><td class='num' data-sort='14.2'>14.2</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1212.0'>1212<span class='sub'>+39 / −41</span></td><td class='num' data-sort='13.5'>13.5</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+38 / −38</span></td><td class='num' data-sort='12.6'>12.6</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+33 / −32</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+38 / −35</span></td><td class='num' data-sort='12.9'>12.9</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1158.0'>1158<span class='sub'>+31 / −36</span></td><td class='num' data-sort='11.3'>11.3</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1155.0'>1155<span class='sub'>+44 / −49</span></td><td class='num' data-sort='6.3'>6.3</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1890.2'>1,890.2</td><td class='num' data-sort='44.9'>44.9</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1155.0'>1155<span class='sub'>+35 / −36</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1134.0'>1134<span class='sub'>+34 / −34</span></td><td class='num' data-sort='11.6'>11.6</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1127.0'>1127<span class='sub'>+38 / −36</span></td><td class='num' data-sort='5.0'>5.0</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='246.0'>246.0</td><td class='num' data-sort='12.3'>12.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1124.0'>1124<span class='sub'>+33 / −36</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2771.3'>2,771.3</td><td class='num' data-sort='48.7'>48.7</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+40 / −46</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='929.0'>929<span class='sub'>+52 / −60</span></td><td class='num' data-sort='-89.9'>-89.9</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1288.0,1248.0,1331.0,18.0,0.661,0.6,0.2,0.0
39
  2,CART,Non-search,Tree,1232.0,1201.0,1272.0,14.4,0.402,0.507,0.1,0.0
40
  3,MI,Non-search,Information theory,1218.0,1183.0,1256.0,14.2,0.649,0.723,0.4,0.0
data/subsets/tasks_all/models_all/budgets_3/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='170.6' x2='170.6' y1='16.2' y2='368.0'/><text class='sa-text' x='170.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='269.2' x2='269.2' y1='16.2' y2='368.0'/><text class='sa-text' x='269.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='367.7' x2='367.7' y1='16.2' y2='368.0'/><text class='sa-text' x='367.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='466.3' x2='466.3' y1='16.2' y2='368.0'/><text class='sa-text' x='466.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='564.8' x2='564.8' y1='16.2' y2='368.0'/><text class='sa-text' x='564.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.4' x2='663.4' y1='16.2' y2='368.0'/><text class='sa-text' x='663.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.9' x2='761.9' y1='16.2' y2='368.0'/><text class='sa-text' x='761.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='81.3' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='72.3' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='81.3' x2='847.8' y1='307.5' y2='307.5'/><text class='sa-text' x='72.3' y='313.5' text-anchor='end' font-size='15'>2.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='263.0' y2='263.0'/><text class='sa-text' x='72.3' y='269.0' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='218.4' y2='218.4'/><text class='sa-text' x='72.3' y='224.4' text-anchor='end' font-size='15'>7.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='173.9' y2='173.9'/><text class='sa-text' x='72.3' y='179.9' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='81.3' x2='847.8' y1='129.3' y2='129.3'/><text class='sa-text' x='72.3' y='135.3' text-anchor='end' font-size='15'>12.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='84.8' y2='84.8'/><text class='sa-text' x='72.3' y='90.8' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='81.3' x2='847.8' y1='40.2' y2='40.2'/><text class='sa-text' x='72.3' y='46.2' text-anchor='end' font-size='15'>17.5</text><rect class='sa-frame-line' x='81.3' y='16.2' width='766.5' height='351.8'/><text class='sa-text' x='464.5' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='123.6' y1='51.9' x2='83.3' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='96.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='116.1,352.0 356.6,108.4 485.6,96.2 531.3,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='531.3' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='485.6' cy='96.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='536.1' cy='99.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='551.8' cy='108.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='356.6' cy='108.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='700.4' cy='112.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='800.2' cy='122.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='700.3' cy='126.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='481.3' cy='144.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='498.8' cy='150.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='813.0' cy='207.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='798.7' cy='239.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='742.3' cy='263.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='116.1' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='531.3' y1='32.2' x2='410.1' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='410.1' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-text sa-halo' x='487.4' y='79.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='536.1' y1='99.7' x2='557.7' y2='89.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='557.7' y='89.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='551.8' y1='108.0' x2='601.9' y2='119.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='601.9' y='119.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='356.6' y1='108.4' x2='383.8' y2='94.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='383.8' y='94.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='700.4' y1='112.0' x2='746.2' y2='97.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='746.2' y='97.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='800.2' y1='122.7' x2='770.7' y2='139.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='770.7' y='139.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='700.3' y1='126.7' x2='670.0' y2='171.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='670.0' y='171.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><text class='sa-dim sa-halo' x='469.6' y='130.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='498.8' y1='150.7' x2='403.0' y2='164.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='403.0' y='164.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='813.0' y1='207.7' x2='828.9' y2='193.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.9' y='193.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='798.7' y1='239.7' x2='820.3' y2='225.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='820.3' y='225.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='742.3' y1='263.7' x2='710.1' y2='277.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='710.1' y='277.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='116.1' y1='352.0' x2='168.9' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='168.9' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='531.3' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 45.7 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 45.7 s&#10;Datasets: 71'/><circle class='sa-hit' cx='485.6' cy='96.2' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 14.4%&#10;Selection time per 1k features: 15.7 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 14.4%&#10;Selection time per 1k features: 15.7 s&#10;Datasets: 71'/><circle class='sa-hit' cx='536.1' cy='99.7' r='10.5' data-tip='MI&#10;Improvement over Random: 14.2%&#10;Selection time per 1k features: 51.1 s&#10;Datasets: 71&#10;Next to mRMR: 13.7%, 73.8 s' aria-label='MI&#10;Improvement over Random: 14.2%&#10;Selection time per 1k features: 51.1 s&#10;Datasets: 71&#10;Next to mRMR: 13.7%, 73.8 s'/><circle class='sa-hit' cx='551.8' cy='108.0' r='10.5' data-tip='mRMR&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 73.8 s&#10;Datasets: 71&#10;Next to MI: 14.2%, 51.1 s' aria-label='mRMR&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 73.8 s&#10;Datasets: 71&#10;Next to MI: 14.2%, 51.1 s'/><circle class='sa-hit' cx='356.6' cy='108.4' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 0.77 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 0.77 s&#10;Datasets: 71'/><circle class='sa-hit' cx='700.4' cy='112.0' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 13.5%&#10;Selection time per 1k features: 2.38e+03 s&#10;Datasets: 71&#10;Next to Lasso: 12.6%, 2.37e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 13.5%&#10;Selection time per 1k features: 2.38e+03 s&#10;Datasets: 71&#10;Next to Lasso: 12.6%, 2.37e+03 s'/><circle class='sa-hit' cx='800.2' cy='122.7' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 12.9%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 12.9%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='700.3' cy='126.7' r='10.5' data-tip='Lasso&#10;Improvement over Random: 12.6%&#10;Selection time per 1k features: 2.37e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.5%, 2.38e+03 s' aria-label='Lasso&#10;Improvement over Random: 12.6%&#10;Selection time per 1k features: 2.37e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.5%, 2.38e+03 s'/><circle class='sa-hit' cx='481.3' cy='144.9' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 11.6%&#10;Selection time per 1k features: 14.2 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 11.3%, 21.4 s' aria-label='GainRatio&#10;Improvement over Random: 11.6%&#10;Selection time per 1k features: 14.2 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 11.3%, 21.4 s'/><circle class='sa-hit' cx='498.8' cy='150.7' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 11.3%&#10;Selection time per 1k features: 21.4 s&#10;Datasets: 71&#10;Next to GainRatio: 11.6%, 14.2 s' aria-label='MarkovBlanket&#10;Improvement over Random: 11.3%&#10;Selection time per 1k features: 21.4 s&#10;Datasets: 71&#10;Next to GainRatio: 11.6%, 14.2 s'/><circle class='sa-hit' cx='813.0' cy='207.7' r='10.5' data-tip='SFS&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 3.3e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 3.3e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='798.7' cy='239.7' r='10.5' data-tip='SBE&#10;Improvement over Random: 6.3%&#10;Selection time per 1k features: 2.36e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 6.3%&#10;Selection time per 1k features: 2.36e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='742.3' cy='263.7' r='10.5' data-tip='LOCO&#10;Improvement over Random: 5.0%&#10;Selection time per 1k features: 6.32e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 5.0%&#10;Selection time per 1k features: 6.32e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='116.1' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0028 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0028 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,17.9516,45.6523,71,True
37
  CART,14.3581,15.705,71,True
38
  MI,14.1607,51.1336,71,False
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='170.6' x2='170.6' y1='16.2' y2='368.0'/><text class='sa-text' x='170.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='269.2' x2='269.2' y1='16.2' y2='368.0'/><text class='sa-text' x='269.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='367.7' x2='367.7' y1='16.2' y2='368.0'/><text class='sa-text' x='367.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='466.3' x2='466.3' y1='16.2' y2='368.0'/><text class='sa-text' x='466.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='564.8' x2='564.8' y1='16.2' y2='368.0'/><text class='sa-text' x='564.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.4' x2='663.4' y1='16.2' y2='368.0'/><text class='sa-text' x='663.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.9' x2='761.9' y1='16.2' y2='368.0'/><text class='sa-text' x='761.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='81.3' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='72.3' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='81.3' x2='847.8' y1='307.5' y2='307.5'/><text class='sa-text' x='72.3' y='313.5' text-anchor='end' font-size='15'>2.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='263.0' y2='263.0'/><text class='sa-text' x='72.3' y='269.0' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='218.4' y2='218.4'/><text class='sa-text' x='72.3' y='224.4' text-anchor='end' font-size='15'>7.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='173.9' y2='173.9'/><text class='sa-text' x='72.3' y='179.9' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='81.3' x2='847.8' y1='129.3' y2='129.3'/><text class='sa-text' x='72.3' y='135.3' text-anchor='end' font-size='15'>12.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='84.8' y2='84.8'/><text class='sa-text' x='72.3' y='90.8' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='81.3' x2='847.8' y1='40.2' y2='40.2'/><text class='sa-text' x='72.3' y='46.2' text-anchor='end' font-size='15'>17.5</text><rect class='sa-frame-line' x='81.3' y='16.2' width='766.5' height='351.8'/><text class='sa-text' x='464.5' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='123.6' y1='51.9' x2='83.3' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='96.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='116.1,352.0 356.6,108.4 485.6,96.2 531.3,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='531.3' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='485.6' cy='96.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='536.1' cy='99.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='551.8' cy='108.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='356.6' cy='108.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='700.4' cy='112.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='800.2' cy='122.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='700.3' cy='126.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='481.3' cy='144.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='498.8' cy='150.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='813.0' cy='207.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='798.7' cy='239.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='742.3' cy='263.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='116.1' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='531.3' y1='32.2' x2='410.7' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='410.7' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-text sa-halo' x='491.2' y='81.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><text class='sa-dim sa-halo' x='554.0' y='89.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='551.8' y1='108.0' x2='601.9' y2='120.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='601.9' y='120.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='356.6' y1='108.4' x2='383.8' y2='94.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='383.8' y='94.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='700.4' y1='112.0' x2='746.2' y2='97.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='746.2' y='97.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='800.2' y1='122.7' x2='773.8' y2='140.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='773.8' y='140.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='700.3' y1='126.7' x2='672.5' y2='169.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='672.5' y='169.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><text class='sa-dim sa-halo' x='473.9' y='130.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='498.8' y1='150.7' x2='403.0' y2='164.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='403.0' y='164.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='813.0' y1='207.7' x2='828.9' y2='193.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.9' y='193.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='798.7' y1='239.7' x2='820.3' y2='225.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='820.3' y='225.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='742.3' y1='263.7' x2='710.1' y2='277.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='710.1' y='277.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='116.1' y1='352.0' x2='168.9' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='168.9' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='531.3' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 45.7 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 45.7 s&#10;Datasets: 71'/><circle class='sa-hit' cx='485.6' cy='96.2' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 14.4%&#10;Selection time per 1k features: 15.7 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 14.4%&#10;Selection time per 1k features: 15.7 s&#10;Datasets: 71'/><circle class='sa-hit' cx='536.1' cy='99.7' r='10.5' data-tip='MI&#10;Improvement over Random: 14.2%&#10;Selection time per 1k features: 51.1 s&#10;Datasets: 71&#10;Next to mRMR: 13.7%, 73.8 s' aria-label='MI&#10;Improvement over Random: 14.2%&#10;Selection time per 1k features: 51.1 s&#10;Datasets: 71&#10;Next to mRMR: 13.7%, 73.8 s'/><circle class='sa-hit' cx='551.8' cy='108.0' r='10.5' data-tip='mRMR&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 73.8 s&#10;Datasets: 71&#10;Next to MI: 14.2%, 51.1 s' aria-label='mRMR&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 73.8 s&#10;Datasets: 71&#10;Next to MI: 14.2%, 51.1 s'/><circle class='sa-hit' cx='356.6' cy='108.4' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 0.77 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 0.77 s&#10;Datasets: 71'/><circle class='sa-hit' cx='700.4' cy='112.0' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 13.5%&#10;Selection time per 1k features: 2.38e+03 s&#10;Datasets: 71&#10;Next to Lasso: 12.6%, 2.37e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 13.5%&#10;Selection time per 1k features: 2.38e+03 s&#10;Datasets: 71&#10;Next to Lasso: 12.6%, 2.37e+03 s'/><circle class='sa-hit' cx='800.2' cy='122.7' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 12.9%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 12.9%&#10;Selection time per 1k features: 2.44e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='700.3' cy='126.7' r='10.5' data-tip='Lasso&#10;Improvement over Random: 12.6%&#10;Selection time per 1k features: 2.37e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.5%, 2.38e+03 s' aria-label='Lasso&#10;Improvement over Random: 12.6%&#10;Selection time per 1k features: 2.37e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.5%, 2.38e+03 s'/><circle class='sa-hit' cx='481.3' cy='144.9' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 11.6%&#10;Selection time per 1k features: 14.2 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 11.3%, 21.4 s' aria-label='GainRatio&#10;Improvement over Random: 11.6%&#10;Selection time per 1k features: 14.2 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 11.3%, 21.4 s'/><circle class='sa-hit' cx='498.8' cy='150.7' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 11.3%&#10;Selection time per 1k features: 21.4 s&#10;Datasets: 71&#10;Next to GainRatio: 11.6%, 14.2 s' aria-label='MarkovBlanket&#10;Improvement over Random: 11.3%&#10;Selection time per 1k features: 21.4 s&#10;Datasets: 71&#10;Next to GainRatio: 11.6%, 14.2 s'/><circle class='sa-hit' cx='813.0' cy='207.7' r='10.5' data-tip='SFS&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 3.3e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 3.3e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='798.7' cy='239.7' r='10.5' data-tip='SBE&#10;Improvement over Random: 6.3%&#10;Selection time per 1k features: 2.36e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 6.3%&#10;Selection time per 1k features: 2.36e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='742.3' cy='263.7' r='10.5' data-tip='LOCO&#10;Improvement over Random: 5.0%&#10;Selection time per 1k features: 6.32e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 5.0%&#10;Selection time per 1k features: 6.32e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='116.1' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0028 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0028 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,17.9516,45.6523,71,True
37
  CART,14.3581,15.705,71,True
38
  MI,14.1607,51.1336,71,False
data/subsets/tasks_all/models_all/budgets_3/per_model_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
data/subsets/tasks_all/models_all/budgets_4/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1250.0'>1250<span class='sub'>+34 / −29</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+35 / −34</span></td><td class='num' data-sort='7.6'>7.6</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+34 / −34</span></td><td class='num' data-sort='8.7'>8.7</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='7.7'>7.7</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+29 / −34</span></td><td class='num' data-sort='7.4'>7.4</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+38 / −39</span></td><td class='num' data-sort='9.9'>9.9</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1165.0'>1165<span class='sub'>+38 / −35</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='14.4'>14.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1140.0'>1140<span class='sub'>+36 / −33</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1131.0'>1131<span class='sub'>+34 / −36</span></td><td class='num' data-sort='9.0'>9.0</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.6'>0.6</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1115.0'>1115<span class='sub'>+36 / −38</span></td><td class='num' data-sort='6.5'>6.5</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1113.0'>1113<span class='sub'>+37 / −40</span></td><td class='num' data-sort='3.9'>3.9</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='989.9'>989.9</td><td class='num' data-sort='41.5'>41.5</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+33 / −32</span></td><td class='num' data-sort='4.4'>4.4</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='243.3'>243.3</td><td class='num' data-sort='18.7'>18.7</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+35 / −33</span></td><td class='num' data-sort='6.0'>6.0</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1092.0'>1092<span class='sub'>+30 / −33</span></td><td class='num' data-sort='3.4'>3.4</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='2953.5'>2,953.5</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+40 / −42</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='980.0'>980<span class='sub'>+46 / −56</span></td><td class='num' data-sort='-92.1'>-92.1</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1250.0,1221.0,1284.0,11.1,0.597,0.507,0.2,0.0
37
  2,CART,Non-search,Tree,1189.0,1155.0,1224.0,7.6,0.324,0.482,0.1,0.0
38
  3,ElasticNet,Non-search,Regularization,1172.0,1138.0,1206.0,8.7,0.308,0.568,7.7,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1250.0'>1250<span class='sub'>+34 / −29</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+35 / −34</span></td><td class='num' data-sort='7.6'>7.6</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+34 / −34</span></td><td class='num' data-sort='8.7'>8.7</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='7.7'>7.7</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+29 / −34</span></td><td class='num' data-sort='7.4'>7.4</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+38 / −39</span></td><td class='num' data-sort='9.9'>9.9</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1165.0'>1165<span class='sub'>+38 / −35</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='14.4'>14.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1140.0'>1140<span class='sub'>+36 / −33</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1131.0'>1131<span class='sub'>+34 / −36</span></td><td class='num' data-sort='9.0'>9.0</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.6'>0.6</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1115.0'>1115<span class='sub'>+36 / −38</span></td><td class='num' data-sort='6.5'>6.5</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1113.0'>1113<span class='sub'>+37 / −40</span></td><td class='num' data-sort='3.9'>3.9</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='989.9'>989.9</td><td class='num' data-sort='41.5'>41.5</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+33 / −32</span></td><td class='num' data-sort='4.4'>4.4</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='243.3'>243.3</td><td class='num' data-sort='18.7'>18.7</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+35 / −33</span></td><td class='num' data-sort='6.0'>6.0</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1092.0'>1092<span class='sub'>+30 / −33</span></td><td class='num' data-sort='3.4'>3.4</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='2953.5'>2,953.5</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+40 / −42</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='980.0'>980<span class='sub'>+46 / −56</span></td><td class='num' data-sort='-92.1'>-92.1</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1250.0,1221.0,1284.0,11.1,0.597,0.507,0.2,0.0
39
  2,CART,Non-search,Tree,1189.0,1155.0,1224.0,7.6,0.324,0.482,0.1,0.0
40
  3,ElasticNet,Non-search,Regularization,1172.0,1138.0,1206.0,8.7,0.308,0.568,7.7,0.0
data/subsets/tasks_all/models_all/budgets_4/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='106.3' x2='106.3' y1='16.2' y2='368.0'/><text class='sa-text' x='106.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='214.2' x2='214.2' y1='16.2' y2='368.0'/><text class='sa-text' x='214.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='322.2' x2='322.2' y1='16.2' y2='368.0'/><text class='sa-text' x='322.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='430.1' x2='430.1' y1='16.2' y2='368.0'/><text class='sa-text' x='430.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='538.0' x2='538.0' y1='16.2' y2='368.0'/><text class='sa-text' x='538.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='645.9' x2='645.9' y1='16.2' y2='368.0'/><text class='sa-text' x='645.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='753.8' x2='753.8' y1='16.2' y2='368.0'/><text class='sa-text' x='753.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='294.5' y2='294.5'/><text class='sa-text' x='57.9' y='300.5' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='236.9' y2='236.9'/><text class='sa-text' x='57.9' y='242.9' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='179.3' y2='179.3'/><text class='sa-text' x='57.9' y='185.3' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='121.7' y2='121.7'/><text class='sa-text' x='57.9' y='127.7' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='64.2' y2='64.2'/><text class='sa-text' x='57.9' y='70.2' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 314.7,65.8 500.7,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='500.7' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='314.7' cy='65.8' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='549.3' cy='92.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.2' cy='102.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.8' cy='120.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='451.4' cy='133.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.6' cy='137.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='506.8' cy='138.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='463.5' cy='163.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='446.4' cy='178.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='731.9' cy='226.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='775.2' cy='240.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='253.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='500.7' y1='32.2' x2='394.3' y2='46.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='394.3' y='46.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='314.7' y1='65.8' x2='275.0' y2='80.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='275.0' y='80.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='549.3' y1='92.6' x2='577.0' y2='78.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='577.0' y='78.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='686.2' y1='102.8' x2='732.7' y2='88.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='732.7' y='88.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='796.9' y='133.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='451.4' y1='133.6' x2='472.8' y2='118.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='472.8' y='118.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='686.6' y1='137.3' x2='744.0' y2='163.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='744.0' y='163.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='506.8' y1='138.8' x2='537.6' y2='120.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='537.6' y='120.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='463.5' y1='163.8' x2='579.7' y2='156.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='579.7' y='156.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='446.4' y1='178.9' x2='382.2' y2='192.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='382.2' y='192.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='731.9' y1='226.8' x2='755.2' y2='212.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='755.2' y='212.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='775.2' y1='240.1' x2='748.1' y2='253.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='748.1' y='253.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='812.3' y1='253.9' x2='828.6' y2='267.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='267.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='500.7' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 45.1 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 45.1 s&#10;Datasets: 71'/><circle class='sa-hit' cx='314.7' cy='65.8' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 9.9%&#10;Selection time per 1k features: 0.852 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 9.9%&#10;Selection time per 1k features: 0.852 s&#10;Datasets: 71'/><circle class='sa-hit' cx='549.3' cy='92.6' r='10.5' data-tip='mRMR&#10;Improvement over Random: 9.0%&#10;Selection time per 1k features: 127 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 9.0%&#10;Selection time per 1k features: 127 s&#10;Datasets: 71'/><circle class='sa-hit' cx='686.2' cy='102.8' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 8.7%&#10;Selection time per 1k features: 2.36e+03 s&#10;Datasets: 71' aria-label='ElasticNet&#10;Improvement over Random: 8.7%&#10;Selection time per 1k features: 2.36e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.8' cy='120.1' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 2.45e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 2.45e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='451.4' cy='133.6' r='10.5' data-tip='CART&#10;Improvement over Random: 7.6%&#10;Selection time per 1k features: 15.8 s&#10;Datasets: 71' aria-label='CART&#10;Improvement over Random: 7.6%&#10;Selection time per 1k features: 15.8 s&#10;Datasets: 71'/><circle class='sa-hit' cx='686.6' cy='137.3' r='10.5' data-tip='Lasso&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 2.38e+03 s&#10;Datasets: 71' aria-label='Lasso&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 2.38e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='506.8' cy='138.8' r='10.5' data-tip='MI&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 51.3 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 51.3 s&#10;Datasets: 71'/><circle class='sa-hit' cx='463.5' cy='163.8' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 20.4 s&#10;Datasets: 71' aria-label='MarkovBlanket&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 20.4 s&#10;Datasets: 71'/><circle class='sa-hit' cx='446.4' cy='178.9' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 6.0%&#10;Selection time per 1k features: 14.2 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 6.0%&#10;Selection time per 1k features: 14.2 s&#10;Datasets: 71'/><circle class='sa-hit' cx='731.9' cy='226.8' r='10.5' data-tip='LOCO&#10;Improvement over Random: 4.4%&#10;Selection time per 1k features: 6.27e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 4.4%&#10;Selection time per 1k features: 6.27e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='775.2' cy='240.1' r='10.5' data-tip='SBE&#10;Improvement over Random: 3.9%&#10;Selection time per 1k features: 1.58e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 3.9%&#10;Selection time per 1k features: 1.58e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='253.9' r='10.5' data-tip='SFS&#10;Improvement over Random: 3.4%&#10;Selection time per 1k features: 3.48e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 3.4%&#10;Selection time per 1k features: 3.48e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0092 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0092 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,11.1111,45.1078,71,True
37
  F-Test,9.9441,0.8521,71,True
38
  mRMR,9.0137,127.2752,71,False
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='106.3' x2='106.3' y1='16.2' y2='368.0'/><text class='sa-text' x='106.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='214.2' x2='214.2' y1='16.2' y2='368.0'/><text class='sa-text' x='214.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='322.2' x2='322.2' y1='16.2' y2='368.0'/><text class='sa-text' x='322.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='430.1' x2='430.1' y1='16.2' y2='368.0'/><text class='sa-text' x='430.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='538.0' x2='538.0' y1='16.2' y2='368.0'/><text class='sa-text' x='538.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='645.9' x2='645.9' y1='16.2' y2='368.0'/><text class='sa-text' x='645.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='753.8' x2='753.8' y1='16.2' y2='368.0'/><text class='sa-text' x='753.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='294.5' y2='294.5'/><text class='sa-text' x='57.9' y='300.5' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='236.9' y2='236.9'/><text class='sa-text' x='57.9' y='242.9' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='179.3' y2='179.3'/><text class='sa-text' x='57.9' y='185.3' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='121.7' y2='121.7'/><text class='sa-text' x='57.9' y='127.7' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='64.2' y2='64.2'/><text class='sa-text' x='57.9' y='70.2' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 314.7,65.8 500.7,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='500.7' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='314.7' cy='65.8' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='549.3' cy='92.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.2' cy='102.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.8' cy='120.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='451.4' cy='133.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.6' cy='137.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='506.8' cy='138.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='463.5' cy='163.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='446.4' cy='178.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='731.9' cy='226.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='775.2' cy='240.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='253.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='500.7' y1='32.2' x2='396.8' y2='45.9' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='396.8' y='45.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='314.7' y1='65.8' x2='276.2' y2='79.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='276.2' y='79.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='549.3' y1='92.6' x2='577.0' y2='78.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='577.0' y='78.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='686.2' y1='102.8' x2='732.7' y2='88.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='732.7' y='88.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='796.9' y='134.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='451.4' y1='133.6' x2='472.8' y2='118.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='472.8' y='118.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='686.6' y1='137.3' x2='741.5' y2='161.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='741.5' y='161.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='506.8' y1='138.8' x2='538.2' y2='119.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='538.2' y='119.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='463.5' y1='163.8' x2='579.0' y2='158.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='579.0' y='158.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='446.4' y1='178.9' x2='380.9' y2='193.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='380.9' y='193.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='731.9' y1='226.8' x2='755.2' y2='212.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='755.2' y='212.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='775.2' y1='240.1' x2='747.5' y2='254.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='747.5' y='254.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='812.3' y1='253.9' x2='828.6' y2='268.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='268.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='500.7' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 45.1 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 45.1 s&#10;Datasets: 71'/><circle class='sa-hit' cx='314.7' cy='65.8' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 9.9%&#10;Selection time per 1k features: 0.852 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 9.9%&#10;Selection time per 1k features: 0.852 s&#10;Datasets: 71'/><circle class='sa-hit' cx='549.3' cy='92.6' r='10.5' data-tip='mRMR&#10;Improvement over Random: 9.0%&#10;Selection time per 1k features: 127 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 9.0%&#10;Selection time per 1k features: 127 s&#10;Datasets: 71'/><circle class='sa-hit' cx='686.2' cy='102.8' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 8.7%&#10;Selection time per 1k features: 2.36e+03 s&#10;Datasets: 71' aria-label='ElasticNet&#10;Improvement over Random: 8.7%&#10;Selection time per 1k features: 2.36e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.8' cy='120.1' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 2.45e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 2.45e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='451.4' cy='133.6' r='10.5' data-tip='CART&#10;Improvement over Random: 7.6%&#10;Selection time per 1k features: 15.8 s&#10;Datasets: 71' aria-label='CART&#10;Improvement over Random: 7.6%&#10;Selection time per 1k features: 15.8 s&#10;Datasets: 71'/><circle class='sa-hit' cx='686.6' cy='137.3' r='10.5' data-tip='Lasso&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 2.38e+03 s&#10;Datasets: 71' aria-label='Lasso&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 2.38e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='506.8' cy='138.8' r='10.5' data-tip='MI&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 51.3 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 51.3 s&#10;Datasets: 71'/><circle class='sa-hit' cx='463.5' cy='163.8' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 20.4 s&#10;Datasets: 71' aria-label='MarkovBlanket&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 20.4 s&#10;Datasets: 71'/><circle class='sa-hit' cx='446.4' cy='178.9' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 6.0%&#10;Selection time per 1k features: 14.2 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 6.0%&#10;Selection time per 1k features: 14.2 s&#10;Datasets: 71'/><circle class='sa-hit' cx='731.9' cy='226.8' r='10.5' data-tip='LOCO&#10;Improvement over Random: 4.4%&#10;Selection time per 1k features: 6.27e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 4.4%&#10;Selection time per 1k features: 6.27e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='775.2' cy='240.1' r='10.5' data-tip='SBE&#10;Improvement over Random: 3.9%&#10;Selection time per 1k features: 1.58e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 3.9%&#10;Selection time per 1k features: 1.58e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='253.9' r='10.5' data-tip='SFS&#10;Improvement over Random: 3.4%&#10;Selection time per 1k features: 3.48e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 3.4%&#10;Selection time per 1k features: 3.48e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0092 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0092 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,11.1111,45.1078,71,True
37
  F-Test,9.9441,0.8521,71,True
38
  mRMR,9.0137,127.2752,71,False
data/subsets/tasks_all/models_all/budgets_4/per_model_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
data/subsets/tasks_all/models_all/budgets_all/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1310.0'>1310<span class='sub'>+32 / −31</span></td><td class='num' data-sort='26.2'>26.2</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1265.0'>1265<span class='sub'>+34 / −29</span></td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+32 / −33</span></td><td class='num' data-sort='21.9'>21.9</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1248.0'>1248<span class='sub'>+28 / −24</span></td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1229.0'>1229<span class='sub'>+28 / −29</span></td><td class='num' data-sort='22.8'>22.8</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1226.0'>1226<span class='sub'>+29 / −32</span></td><td class='num' data-sort='20.9'>20.9</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='2.1'>2.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1220.0'>1220<span class='sub'>+27 / −28</span></td><td class='num' data-sort='21.2'>21.2</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1198.0'>1198<span class='sub'>+32 / −32</span></td><td class='num' data-sort='18.6'>18.6</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1193.0'>1193<span class='sub'>+27 / −28</span></td><td class='num' data-sort='21.7'>21.7</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1183.0'>1183<span class='sub'>+28 / −26</span></td><td class='num' data-sort='19.5'>19.5</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1178.0'>1178<span class='sub'>+31 / −32</span></td><td class='num' data-sort='18.2'>18.2</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2105.8'>2,105.8</td><td class='num' data-sort='43.5'>43.5</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+38 / −39</span></td><td class='num' data-sort='11.4'>11.4</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2495.5'>2,495.5</td><td class='num' data-sort='46.5'>46.5</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1151.0'>1151<span class='sub'>+33 / −32</span></td><td class='num' data-sort='13.0'>13.0</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='241.5'>241.5</td><td class='num' data-sort='7.6'>7.6</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+32 / −34</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='965.0'>965<span class='sub'>+43 / −52</span></td><td class='num' data-sort='-18.6'>-18.6</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1310.0,1279.0,1342.0,26.2,0.643,0.669,0.2,0.0
37
  2,CART,Non-search,Tree,1265.0,1236.0,1299.0,23.8,0.422,0.586,0.1,0.0
38
  3,ElasticNet,Non-search,Regularization,1252.0,1219.0,1284.0,21.9,0.447,0.76,7.6,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1310.0'>1310<span class='sub'>+32 / −31</span></td><td class='num' data-sort='26.2'>26.2</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1265.0'>1265<span class='sub'>+34 / −29</span></td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+32 / −33</span></td><td class='num' data-sort='21.9'>21.9</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1248.0'>1248<span class='sub'>+28 / −24</span></td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1229.0'>1229<span class='sub'>+28 / −29</span></td><td class='num' data-sort='22.8'>22.8</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1226.0'>1226<span class='sub'>+29 / −32</span></td><td class='num' data-sort='20.9'>20.9</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='2.1'>2.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1220.0'>1220<span class='sub'>+27 / −28</span></td><td class='num' data-sort='21.2'>21.2</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1198.0'>1198<span class='sub'>+32 / −32</span></td><td class='num' data-sort='18.6'>18.6</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1193.0'>1193<span class='sub'>+27 / −28</span></td><td class='num' data-sort='21.7'>21.7</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1183.0'>1183<span class='sub'>+28 / −26</span></td><td class='num' data-sort='19.5'>19.5</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1178.0'>1178<span class='sub'>+31 / −32</span></td><td class='num' data-sort='18.2'>18.2</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2105.8'>2,105.8</td><td class='num' data-sort='43.5'>43.5</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+38 / −39</span></td><td class='num' data-sort='11.4'>11.4</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2495.5'>2,495.5</td><td class='num' data-sort='46.5'>46.5</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1151.0'>1151<span class='sub'>+33 / −32</span></td><td class='num' data-sort='13.0'>13.0</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='241.5'>241.5</td><td class='num' data-sort='7.6'>7.6</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+32 / −34</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='965.0'>965<span class='sub'>+43 / −52</span></td><td class='num' data-sort='-18.6'>-18.6</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1310.0,1279.0,1342.0,26.2,0.643,0.669,0.2,0.0
39
  2,CART,Non-search,Tree,1265.0,1236.0,1299.0,23.8,0.422,0.586,0.1,0.0
40
  3,ElasticNet,Non-search,Regularization,1252.0,1219.0,1284.0,21.9,0.447,0.76,7.6,0.0
data/subsets/tasks_all/models_all/budgets_all/per_model_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
data/subsets/tasks_all/models_lightgbm/budgets_0/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1368.0'>1368<span class='sub'>+66 / −57</span></td><td class='num' data-sort='29.5'>29.5</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1347.0'>1347<span class='sub'>+56 / −45</span></td><td class='num' data-sort='27.7'>27.7</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+55 / −50</span></td><td class='num' data-sort='25.8'>25.8</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1312.0'>1312<span class='sub'>+56 / −56</span></td><td class='num' data-sort='24.2'>24.2</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='1052.3'>1,052.3</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1304.0'>1304<span class='sub'>+46 / −46</span></td><td class='num' data-sort='25.2'>25.2</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1292.0'>1292<span class='sub'>+43 / −42</span></td><td class='num' data-sort='27.8'>27.8</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1280.0'>1280<span class='sub'>+48 / −52</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1270.0'>1270<span class='sub'>+43 / −49</span></td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1254.0'>1254<span class='sub'>+45 / −44</span></td><td class='num' data-sort='25.6'>25.6</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1234.0'>1234<span class='sub'>+50 / −49</span></td><td class='num' data-sort='23.3'>23.3</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1233.0'>1233<span class='sub'>+56 / −56</span></td><td class='num' data-sort='14.6'>14.6</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3579.1'>3,579.1</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1194.0'>1194<span class='sub'>+47 / −47</span></td><td class='num' data-sort='16.2'>16.2</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='240.9'>240.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1191.0'>1191<span class='sub'>+44 / −43</span></td><td class='num' data-sort='19.7'>19.7</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+54 / −67</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='934.0'>934<span class='sub'>+75 / −97</span></td><td class='num' data-sort='-19.0'>-19.0</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1368.0,1311.0,1434.0,29.5,0.628,0.805,0.2,0.0
37
  2,CART,Non-search,Tree,1347.0,1302.0,1403.0,27.7,0.484,0.731,0.1,0.0
38
  3,MarkovBlanket,Search,Backward search,1325.0,1275.0,1380.0,25.8,0.418,0.465,0.1,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1368.0'>1368<span class='sub'>+66 / −57</span></td><td class='num' data-sort='29.5'>29.5</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1347.0'>1347<span class='sub'>+56 / −45</span></td><td class='num' data-sort='27.7'>27.7</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+55 / −50</span></td><td class='num' data-sort='25.8'>25.8</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1312.0'>1312<span class='sub'>+56 / −56</span></td><td class='num' data-sort='24.2'>24.2</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='1052.3'>1,052.3</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1304.0'>1304<span class='sub'>+46 / −46</span></td><td class='num' data-sort='25.2'>25.2</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1292.0'>1292<span class='sub'>+43 / −42</span></td><td class='num' data-sort='27.8'>27.8</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1280.0'>1280<span class='sub'>+48 / −52</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1270.0'>1270<span class='sub'>+43 / −49</span></td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1254.0'>1254<span class='sub'>+45 / −44</span></td><td class='num' data-sort='25.6'>25.6</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1234.0'>1234<span class='sub'>+50 / −49</span></td><td class='num' data-sort='23.3'>23.3</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1233.0'>1233<span class='sub'>+56 / −56</span></td><td class='num' data-sort='14.6'>14.6</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3579.1'>3,579.1</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1194.0'>1194<span class='sub'>+47 / −47</span></td><td class='num' data-sort='16.2'>16.2</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='240.9'>240.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1191.0'>1191<span class='sub'>+44 / −43</span></td><td class='num' data-sort='19.7'>19.7</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+54 / −67</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='934.0'>934<span class='sub'>+75 / −97</span></td><td class='num' data-sort='-19.0'>-19.0</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1368.0,1311.0,1434.0,29.5,0.628,0.805,0.2,0.0
39
  2,CART,Non-search,Tree,1347.0,1302.0,1403.0,27.7,0.484,0.731,0.1,0.0
40
  3,MarkovBlanket,Search,Backward search,1325.0,1275.0,1380.0,25.8,0.418,0.465,0.1,0.0
data/subsets/tasks_all/models_lightgbm/budgets_0/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='157.8' x2='157.8' y1='16.2' y2='368.0'/><text class='sa-text' x='157.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='257.7' x2='257.7' y1='16.2' y2='368.0'/><text class='sa-text' x='257.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='357.7' x2='357.7' y1='16.2' y2='368.0'/><text class='sa-text' x='357.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='457.7' x2='457.7' y1='16.2' y2='368.0'/><text class='sa-text' x='457.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='557.6' x2='557.6' y1='16.2' y2='368.0'/><text class='sa-text' x='557.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.6' x2='657.6' y1='16.2' y2='368.0'/><text class='sa-text' x='657.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.6' x2='757.6' y1='16.2' y2='368.0'/><text class='sa-text' x='757.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='297.9' y2='297.9'/><text class='sa-text' x='58.0' y='303.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='243.8' y2='243.8'/><text class='sa-text' x='58.0' y='249.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='189.6' y2='189.6'/><text class='sa-text' x='58.0' y='195.6' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='135.5' y2='135.5'/><text class='sa-text' x='58.0' y='141.5' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='81.4' y2='81.4'/><text class='sa-text' x='58.0' y='87.4' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='27.2' y2='27.2'/><text class='sa-text' x='58.0' y='33.2' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 338.9,71.3 473.8,52.4 516.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='523.9' cy='50.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='473.8' cy='52.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='338.9' cy='71.3' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='485.4' cy='73.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='478.9' cy='74.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='690.7' cy='79.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='783.4' cy='90.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.7' cy='93.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.1' cy='100.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.4' cy='138.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.7' cy='176.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='193.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.5' y1='32.2' x2='646.8' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='646.8' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='523.9' y1='50.9' x2='581.9' y2='53.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='581.9' y='53.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='473.8' y1='52.4' x2='468.1' y2='26.2' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='468.1' y='26.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='338.9' y1='71.3' x2='309.3' y2='92.4' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='309.3' y='92.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='485.4' y1='73.0' x2='578.0' y2='91.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='578.0' y='91.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='478.9' y1='74.8' x2='419.7' y2='54.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='419.7' y='54.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='690.7' y1='79.5' x2='785.7' y2='59.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='785.7' y='59.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='783.4' y1='90.3' x2='812.4' y2='102.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='812.4' y='102.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='686.7' y1='93.8' x2='735.5' y2='120.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='735.5' y='120.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><text class='sa-dim sa-halo' x='465.0' y='120.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='797.0' y='152.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='737.7' y1='176.5' x2='704.9' y2='190.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='704.9' y='190.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='207.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.5%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71&#10;Next to MI: 27.8%, 46 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.5%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71&#10;Next to MI: 27.8%, 46 s'/><circle class='sa-hit' cx='523.9' cy='50.9' r='10.5' data-tip='MI&#10;Improvement over Random: 27.8%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71&#10;Next to RFImportance: 29.5%, 38.8 s' aria-label='MI&#10;Improvement over Random: 27.8%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71&#10;Next to RFImportance: 29.5%, 38.8 s'/><circle class='sa-hit' cx='473.8' cy='52.4' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 27.7%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 27.7%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='338.9' cy='71.3' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 0.648 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 0.648 s&#10;Datasets: 71'/><circle class='sa-hit' cx='485.4' cy='73.0' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 25.8%&#10;Selection time per 1k features: 19 s&#10;Datasets: 71&#10;Next to mRMR: 25.6%, 16.3 s' aria-label='MarkovBlanket&#10;Improvement over Random: 25.8%&#10;Selection time per 1k features: 19 s&#10;Datasets: 71&#10;Next to mRMR: 25.6%, 16.3 s'/><circle class='sa-hit' cx='478.9' cy='74.8' r='10.5' data-tip='mRMR&#10;Improvement over Random: 25.6%&#10;Selection time per 1k features: 16.3 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 25.8%, 19 s' aria-label='mRMR&#10;Improvement over Random: 25.6%&#10;Selection time per 1k features: 16.3 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 25.8%, 19 s'/><circle class='sa-hit' cx='690.7' cy='79.5' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 25.2%&#10;Selection time per 1k features: 2.14e+03 s&#10;Datasets: 71&#10;Next to Lasso: 23.8%, 1.96e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 25.2%&#10;Selection time per 1k features: 2.14e+03 s&#10;Datasets: 71&#10;Next to Lasso: 23.8%, 1.96e+03 s'/><circle class='sa-hit' cx='783.4' cy='90.3' r='10.5' data-tip='SFS&#10;Improvement over Random: 24.2%&#10;Selection time per 1k features: 1.81e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 24.2%&#10;Selection time per 1k features: 1.81e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='686.7' cy='93.8' r='10.5' data-tip='Lasso&#10;Improvement over Random: 23.8%&#10;Selection time per 1k features: 1.96e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.2%, 2.14e+03 s' aria-label='Lasso&#10;Improvement over Random: 23.8%&#10;Selection time per 1k features: 1.96e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.2%, 2.14e+03 s'/><circle class='sa-hit' cx='468.1' cy='100.2' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.4' cy='138.5' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 19.7%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 19.7%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='737.7' cy='176.5' r='10.5' data-tip='LOCO&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 6.32e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 6.32e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='193.5' r='10.5' data-tip='SBE&#10;Improvement over Random: 14.6%&#10;Selection time per 1k features: 3.53e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 14.6%&#10;Selection time per 1k features: 3.53e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0028 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0028 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,29.5432,38.7674,71,True
37
  MI,27.8197,45.9925,71,False
38
  CART,27.6757,14.4968,71,True
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='157.8' x2='157.8' y1='16.2' y2='368.0'/><text class='sa-text' x='157.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='257.7' x2='257.7' y1='16.2' y2='368.0'/><text class='sa-text' x='257.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='357.7' x2='357.7' y1='16.2' y2='368.0'/><text class='sa-text' x='357.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='457.7' x2='457.7' y1='16.2' y2='368.0'/><text class='sa-text' x='457.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='557.6' x2='557.6' y1='16.2' y2='368.0'/><text class='sa-text' x='557.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.6' x2='657.6' y1='16.2' y2='368.0'/><text class='sa-text' x='657.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.6' x2='757.6' y1='16.2' y2='368.0'/><text class='sa-text' x='757.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='297.9' y2='297.9'/><text class='sa-text' x='58.0' y='303.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='243.8' y2='243.8'/><text class='sa-text' x='58.0' y='249.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='189.6' y2='189.6'/><text class='sa-text' x='58.0' y='195.6' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='135.5' y2='135.5'/><text class='sa-text' x='58.0' y='141.5' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='81.4' y2='81.4'/><text class='sa-text' x='58.0' y='87.4' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='27.2' y2='27.2'/><text class='sa-text' x='58.0' y='33.2' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 338.9,71.3 473.8,52.4 516.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='523.9' cy='50.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='473.8' cy='52.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='338.9' cy='71.3' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='485.4' cy='73.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='478.9' cy='74.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='690.7' cy='79.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='783.4' cy='90.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.7' cy='93.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.1' cy='100.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.4' cy='138.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.7' cy='176.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='193.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.5' y1='32.2' x2='636.8' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='636.8' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='523.9' y1='50.9' x2='588.1' y2='59.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='588.1' y='59.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='473.8' y1='52.4' x2='466.2' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='466.2' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='338.9' y1='71.3' x2='310.5' y2='93.0' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='310.5' y='93.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='485.4' y1='73.0' x2='584.3' y2='89.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='584.3' y='89.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='478.9' y1='74.8' x2='410.9' y2='59.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='410.9' y='59.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='690.7' y1='79.5' x2='789.5' y2='62.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='789.5' y='62.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='783.4' y1='90.3' x2='811.8' y2='101.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='811.8' y='101.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='686.7' y1='93.8' x2='727.9' y2='123.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='727.9' y='123.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='468.1' y1='100.2' x2='463.7' y2='120.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='463.7' y='120.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='797.0' y='152.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='737.7' y1='176.5' x2='704.3' y2='191.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='704.3' y='191.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='207.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.5%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71&#10;Next to MI: 27.8%, 46 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.5%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71&#10;Next to MI: 27.8%, 46 s'/><circle class='sa-hit' cx='523.9' cy='50.9' r='10.5' data-tip='MI&#10;Improvement over Random: 27.8%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71&#10;Next to RFImportance: 29.5%, 38.8 s' aria-label='MI&#10;Improvement over Random: 27.8%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71&#10;Next to RFImportance: 29.5%, 38.8 s'/><circle class='sa-hit' cx='473.8' cy='52.4' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 27.7%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 27.7%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='338.9' cy='71.3' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 0.648 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 0.648 s&#10;Datasets: 71'/><circle class='sa-hit' cx='485.4' cy='73.0' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 25.8%&#10;Selection time per 1k features: 19 s&#10;Datasets: 71&#10;Next to mRMR: 25.6%, 16.3 s' aria-label='MarkovBlanket&#10;Improvement over Random: 25.8%&#10;Selection time per 1k features: 19 s&#10;Datasets: 71&#10;Next to mRMR: 25.6%, 16.3 s'/><circle class='sa-hit' cx='478.9' cy='74.8' r='10.5' data-tip='mRMR&#10;Improvement over Random: 25.6%&#10;Selection time per 1k features: 16.3 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 25.8%, 19 s' aria-label='mRMR&#10;Improvement over Random: 25.6%&#10;Selection time per 1k features: 16.3 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 25.8%, 19 s'/><circle class='sa-hit' cx='690.7' cy='79.5' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 25.2%&#10;Selection time per 1k features: 2.14e+03 s&#10;Datasets: 71&#10;Next to Lasso: 23.8%, 1.96e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 25.2%&#10;Selection time per 1k features: 2.14e+03 s&#10;Datasets: 71&#10;Next to Lasso: 23.8%, 1.96e+03 s'/><circle class='sa-hit' cx='783.4' cy='90.3' r='10.5' data-tip='SFS&#10;Improvement over Random: 24.2%&#10;Selection time per 1k features: 1.81e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 24.2%&#10;Selection time per 1k features: 1.81e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='686.7' cy='93.8' r='10.5' data-tip='Lasso&#10;Improvement over Random: 23.8%&#10;Selection time per 1k features: 1.96e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.2%, 2.14e+03 s' aria-label='Lasso&#10;Improvement over Random: 23.8%&#10;Selection time per 1k features: 1.96e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.2%, 2.14e+03 s'/><circle class='sa-hit' cx='468.1' cy='100.2' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.4' cy='138.5' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 19.7%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 19.7%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='737.7' cy='176.5' r='10.5' data-tip='LOCO&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 6.32e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 6.32e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='193.5' r='10.5' data-tip='SBE&#10;Improvement over Random: 14.6%&#10;Selection time per 1k features: 3.53e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 14.6%&#10;Selection time per 1k features: 3.53e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0028 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0028 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,29.5432,38.7674,71,True
37
  MI,27.8197,45.9925,71,False
38
  CART,27.6757,14.4968,71,True
data/subsets/tasks_all/models_lightgbm/budgets_1/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+57 / −52</span></td><td class='num' data-sort='28.1'>28.1</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+49 / −46</span></td><td class='num' data-sort='25.7'>25.7</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1355.0'>1355<span class='sub'>+49 / −48</span></td><td class='num' data-sort='26.5'>26.5</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1340.0'>1340<span class='sub'>+47 / −40</span></td><td class='num' data-sort='24.8'>24.8</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1338.0'>1338<span class='sub'>+48 / −50</span></td><td class='num' data-sort='25.4'>25.4</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.1'>2.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+41 / −36</span></td><td class='num' data-sort='26.8'>26.8</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1311.0'>1311<span class='sub'>+50 / −48</span></td><td class='num' data-sort='25.0'>25.0</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1294.0'>1294<span class='sub'>+51 / −50</span></td><td class='num' data-sort='21.3'>21.3</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1770.9'>1,770.9</td><td class='num' data-sort='44.1'>44.1</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1291.0'>1291<span class='sub'>+47 / −44</span></td><td class='num' data-sort='24.3'>24.3</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1289.0'>1289<span class='sub'>+47 / −48</span></td><td class='num' data-sort='24.5'>24.5</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1275.0'>1275<span class='sub'>+49 / −50</span></td><td class='num' data-sort='21.9'>21.9</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='12.8'>12.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1259.0'>1259<span class='sub'>+55 / −59</span></td><td class='num' data-sort='15.5'>15.5</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3152.6'>3,152.6</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1241.0'>1241<span class='sub'>+55 / −51</span></td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='239.5'>239.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+44 / −53</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='995.0'>995<span class='sub'>+67 / −88</span></td><td class='num' data-sort='-29.1'>-29.1</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1366.0,1314.0,1423.0,28.1,0.653,0.741,0.2,0.0
37
  2,ElasticNet,Non-search,Regularization,1366.0,1320.0,1415.0,25.7,0.519,0.873,6.3,0.0
38
  3,CART,Non-search,Tree,1355.0,1307.0,1404.0,26.5,0.456,0.638,0.0,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+57 / −52</span></td><td class='num' data-sort='28.1'>28.1</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+49 / −46</span></td><td class='num' data-sort='25.7'>25.7</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1355.0'>1355<span class='sub'>+49 / −48</span></td><td class='num' data-sort='26.5'>26.5</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1340.0'>1340<span class='sub'>+47 / −40</span></td><td class='num' data-sort='24.8'>24.8</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1338.0'>1338<span class='sub'>+48 / −50</span></td><td class='num' data-sort='25.4'>25.4</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.1'>2.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+41 / −36</span></td><td class='num' data-sort='26.8'>26.8</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1311.0'>1311<span class='sub'>+50 / −48</span></td><td class='num' data-sort='25.0'>25.0</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1294.0'>1294<span class='sub'>+51 / −50</span></td><td class='num' data-sort='21.3'>21.3</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1770.9'>1,770.9</td><td class='num' data-sort='44.1'>44.1</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1291.0'>1291<span class='sub'>+47 / −44</span></td><td class='num' data-sort='24.3'>24.3</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1289.0'>1289<span class='sub'>+47 / −48</span></td><td class='num' data-sort='24.5'>24.5</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1275.0'>1275<span class='sub'>+49 / −50</span></td><td class='num' data-sort='21.9'>21.9</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='12.8'>12.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1259.0'>1259<span class='sub'>+55 / −59</span></td><td class='num' data-sort='15.5'>15.5</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3152.6'>3,152.6</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1241.0'>1241<span class='sub'>+55 / −51</span></td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='239.5'>239.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+44 / −53</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='995.0'>995<span class='sub'>+67 / −88</span></td><td class='num' data-sort='-29.1'>-29.1</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1366.0,1314.0,1423.0,28.1,0.653,0.741,0.2,0.0
39
  2,ElasticNet,Non-search,Regularization,1366.0,1320.0,1415.0,25.7,0.519,0.873,6.3,0.0
40
  3,CART,Non-search,Tree,1355.0,1307.0,1404.0,26.5,0.456,0.638,0.0,0.0
data/subsets/tasks_all/models_lightgbm/budgets_2/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1321.0'>1321<span class='sub'>+59 / −50</span></td><td class='num' data-sort='22.8'>22.8</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1283.0'>1283<span class='sub'>+48 / −47</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1282.0'>1282<span class='sub'>+49 / −50</span></td><td class='num' data-sort='19.5'>19.5</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1268.0'>1268<span class='sub'>+51 / −51</span></td><td class='num' data-sort='18.7'>18.7</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+49 / −49</span></td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1248.0'>1248<span class='sub'>+38 / −38</span></td><td class='num' data-sort='17.8'>17.8</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1224.0'>1224<span class='sub'>+58 / −53</span></td><td class='num' data-sort='11.4'>11.4</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2887.2'>2,887.2</td><td class='num' data-sort='47.4'>47.4</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1212.0'>1212<span class='sub'>+47 / −45</span></td><td class='num' data-sort='16.6'>16.6</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1205.0'>1205<span class='sub'>+47 / −45</span></td><td class='num' data-sort='16.5'>16.5</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1199.0'>1199<span class='sub'>+44 / −47</span></td><td class='num' data-sort='16.3'>16.3</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='12.9'>12.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+43 / −44</span></td><td class='num' data-sort='16.6'>16.6</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1193.0'>1193<span class='sub'>+51 / −52</span></td><td class='num' data-sort='13.1'>13.1</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2274.8'>2,274.8</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1175.0'>1175<span class='sub'>+55 / −56</span></td><td class='num' data-sort='9.1'>9.1</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='251.9'>251.9</td><td class='num' data-sort='6.6'>6.6</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+46 / −52</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='939.0'>939<span class='sub'>+74 / −91</span></td><td class='num' data-sort='-65.9'>-65.9</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1321.0,1271.0,1380.0,22.8,0.676,0.691,0.2,0.0
37
  2,CART,Non-search,Tree,1283.0,1236.0,1331.0,19.8,0.444,0.572,0.0,0.0
38
  3,MI,Non-search,Information theory,1282.0,1232.0,1331.0,19.5,0.683,0.815,0.4,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1321.0'>1321<span class='sub'>+59 / −50</span></td><td class='num' data-sort='22.8'>22.8</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1283.0'>1283<span class='sub'>+48 / −47</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1282.0'>1282<span class='sub'>+49 / −50</span></td><td class='num' data-sort='19.5'>19.5</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1268.0'>1268<span class='sub'>+51 / −51</span></td><td class='num' data-sort='18.7'>18.7</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+49 / −49</span></td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1248.0'>1248<span class='sub'>+38 / −38</span></td><td class='num' data-sort='17.8'>17.8</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1224.0'>1224<span class='sub'>+58 / −53</span></td><td class='num' data-sort='11.4'>11.4</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2887.2'>2,887.2</td><td class='num' data-sort='47.4'>47.4</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1212.0'>1212<span class='sub'>+47 / −45</span></td><td class='num' data-sort='16.6'>16.6</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1205.0'>1205<span class='sub'>+47 / −45</span></td><td class='num' data-sort='16.5'>16.5</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1199.0'>1199<span class='sub'>+44 / −47</span></td><td class='num' data-sort='16.3'>16.3</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='12.9'>12.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+43 / −44</span></td><td class='num' data-sort='16.6'>16.6</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1193.0'>1193<span class='sub'>+51 / −52</span></td><td class='num' data-sort='13.1'>13.1</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2274.8'>2,274.8</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1175.0'>1175<span class='sub'>+55 / −56</span></td><td class='num' data-sort='9.1'>9.1</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='251.9'>251.9</td><td class='num' data-sort='6.6'>6.6</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+46 / −52</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='939.0'>939<span class='sub'>+74 / −91</span></td><td class='num' data-sort='-65.9'>-65.9</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1321.0,1271.0,1380.0,22.8,0.676,0.691,0.2,0.0
39
  2,CART,Non-search,Tree,1283.0,1236.0,1331.0,19.8,0.444,0.572,0.0,0.0
40
  3,MI,Non-search,Information theory,1282.0,1232.0,1331.0,19.5,0.683,0.815,0.4,0.0
data/subsets/tasks_all/models_lightgbm/budgets_3/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+58 / −49</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1210.0'>1210<span class='sub'>+48 / −41</span></td><td class='num' data-sort='14.3'>14.3</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1207.0'>1207<span class='sub'>+46 / −47</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1207.0'>1207<span class='sub'>+54 / −53</span></td><td class='num' data-sort='13.3'>13.3</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+48 / −49</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1183.0'>1183<span class='sub'>+44 / −48</span></td><td class='num' data-sort='14.2'>14.2</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1153.0'>1153<span class='sub'>+43 / −43</span></td><td class='num' data-sort='13.5'>13.5</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1145.0'>1145<span class='sub'>+52 / −50</span></td><td class='num' data-sort='12.8'>12.8</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='13.1'>13.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1128.0'>1128<span class='sub'>+43 / −43</span></td><td class='num' data-sort='12.3'>12.3</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1122.0'>1122<span class='sub'>+57 / −58</span></td><td class='num' data-sort='5.8'>5.8</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1923.4'>1,923.4</td><td class='num' data-sort='44.6'>44.6</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1118.0'>1118<span class='sub'>+53 / −53</span></td><td class='num' data-sort='6.2'>6.2</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='236.6'>236.6</td><td class='num' data-sort='11.3'>11.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1115.0'>1115<span class='sub'>+46 / −44</span></td><td class='num' data-sort='12.3'>12.3</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1098.0'>1098<span class='sub'>+50 / −48</span></td><td class='num' data-sort='7.3'>7.3</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2771.3'>2,771.3</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+50 / −57</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='878.0'>878<span class='sub'>+72 / −82</span></td><td class='num' data-sort='-126.6'>-126.6</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1273.0,1224.0,1331.0,17.7,0.661,0.6,0.2,0.0
37
  2,CART,Non-search,Tree,1210.0,1169.0,1258.0,14.3,0.402,0.507,0.0,0.0
38
  3,Lasso,Non-search,Regularization,1207.0,1160.0,1253.0,13.7,0.334,0.643,1.9,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+58 / −49</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1210.0'>1210<span class='sub'>+48 / −41</span></td><td class='num' data-sort='14.3'>14.3</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1207.0'>1207<span class='sub'>+46 / −47</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1207.0'>1207<span class='sub'>+54 / −53</span></td><td class='num' data-sort='13.3'>13.3</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+48 / −49</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1183.0'>1183<span class='sub'>+44 / −48</span></td><td class='num' data-sort='14.2'>14.2</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1153.0'>1153<span class='sub'>+43 / −43</span></td><td class='num' data-sort='13.5'>13.5</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1145.0'>1145<span class='sub'>+52 / −50</span></td><td class='num' data-sort='12.8'>12.8</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='13.1'>13.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1128.0'>1128<span class='sub'>+43 / −43</span></td><td class='num' data-sort='12.3'>12.3</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1122.0'>1122<span class='sub'>+57 / −58</span></td><td class='num' data-sort='5.8'>5.8</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1923.4'>1,923.4</td><td class='num' data-sort='44.6'>44.6</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1118.0'>1118<span class='sub'>+53 / −53</span></td><td class='num' data-sort='6.2'>6.2</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='236.6'>236.6</td><td class='num' data-sort='11.3'>11.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1115.0'>1115<span class='sub'>+46 / −44</span></td><td class='num' data-sort='12.3'>12.3</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1098.0'>1098<span class='sub'>+50 / −48</span></td><td class='num' data-sort='7.3'>7.3</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2771.3'>2,771.3</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+50 / −57</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='878.0'>878<span class='sub'>+72 / −82</span></td><td class='num' data-sort='-126.6'>-126.6</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1273.0,1224.0,1331.0,17.7,0.661,0.6,0.2,0.0
39
  2,CART,Non-search,Tree,1210.0,1169.0,1258.0,14.3,0.402,0.507,0.0,0.0
40
  3,Lasso,Non-search,Regularization,1207.0,1160.0,1253.0,13.7,0.334,0.643,1.9,0.0
data/subsets/tasks_all/models_lightgbm/budgets_3/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='173.5' x2='173.5' y1='16.2' y2='368.0'/><text class='sa-text' x='173.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='271.4' x2='271.4' y1='16.2' y2='368.0'/><text class='sa-text' x='271.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='369.4' x2='369.4' y1='16.2' y2='368.0'/><text class='sa-text' x='369.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='467.4' x2='467.4' y1='16.2' y2='368.0'/><text class='sa-text' x='467.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='565.4' x2='565.4' y1='16.2' y2='368.0'/><text class='sa-text' x='565.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.4' x2='663.4' y1='16.2' y2='368.0'/><text class='sa-text' x='663.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.4' x2='761.4' y1='16.2' y2='368.0'/><text class='sa-text' x='761.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='81.3' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='72.3' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='81.3' x2='847.8' y1='306.8' y2='306.8'/><text class='sa-text' x='72.3' y='312.8' text-anchor='end' font-size='15'>2.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='261.6' y2='261.6'/><text class='sa-text' x='72.3' y='267.6' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='216.4' y2='216.4'/><text class='sa-text' x='72.3' y='222.4' text-anchor='end' font-size='15'>7.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='171.1' y2='171.1'/><text class='sa-text' x='72.3' y='177.1' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='81.3' x2='847.8' y1='125.9' y2='125.9'/><text class='sa-text' x='72.3' y='131.9' text-anchor='end' font-size='15'>12.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='80.7' y2='80.7'/><text class='sa-text' x='72.3' y='86.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='81.3' x2='847.8' y1='35.4' y2='35.4'/><text class='sa-text' x='72.3' y='41.4' text-anchor='end' font-size='15'>17.5</text><rect class='sa-frame-line' x='81.3' y='16.2' width='766.5' height='351.8'/><text class='sa-text' x='464.5' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='123.6' y1='51.9' x2='83.3' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='96.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='116.1,352.0 350.8,96.0 483.0,93.4 524.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='524.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='483.0' cy='93.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='350.8' cy='96.0' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='532.7' cy='103.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='692.7' cy='104.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='547.5' cy='108.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='111.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='798.3' cy='120.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='477.6' cy='129.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='494.6' cy='130.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='813.0' cy='220.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='742.2' cy='240.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='799.0' cy='247.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='116.1' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='524.8' y1='32.2' x2='647.2' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='647.2' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='483.0' y1='93.4' x2='494.7' y2='75.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='494.7' y='75.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='350.8' y1='96.0' x2='301.4' y2='106.9' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='301.4' y='106.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='532.7' y1='103.7' x2='563.6' y2='66.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='563.6' y='66.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='692.7' y1='104.6' x2='715.7' y2='89.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='715.7' y='89.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='547.5' y1='108.5' x2='601.8' y2='97.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='601.8' y='97.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='695.4' y1='111.1' x2='720.3' y2='133.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='720.3' y='133.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='798.3' y1='120.0' x2='797.9' y2='161.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='797.9' y='161.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='477.6' y1='129.1' x2='402.2' y2='137.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='402.2' y='137.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='494.6' y1='130.1' x2='581.7' y2='174.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='581.7' y='174.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='813.0' y1='220.6' x2='828.9' y2='206.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.9' y='206.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='742.2' y1='240.7' x2='708.8' y2='255.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='708.8' y='255.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='799.0' y1='247.0' x2='825.0' y2='260.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='825.0' y='260.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='116.1' y1='352.0' x2='168.9' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='168.9' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='524.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='483.0' cy='93.4' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 14.3%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 14.3%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71'/><circle class='sa-hit' cx='350.8' cy='96.0' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 14.2%&#10;Selection time per 1k features: 0.645 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 14.2%&#10;Selection time per 1k features: 0.645 s&#10;Datasets: 71'/><circle class='sa-hit' cx='532.7' cy='103.7' r='10.5' data-tip='MI&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 46.4 s&#10;Datasets: 71&#10;Next to mRMR: 13.5%, 65.6 s' aria-label='MI&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 46.4 s&#10;Datasets: 71&#10;Next to mRMR: 13.5%, 65.6 s'/><circle class='sa-hit' cx='692.7' cy='104.6' r='10.5' data-tip='Lasso&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.3%, 2.12e+03 s' aria-label='Lasso&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.3%, 2.12e+03 s'/><circle class='sa-hit' cx='547.5' cy='108.5' r='10.5' data-tip='mRMR&#10;Improvement over Random: 13.5%&#10;Selection time per 1k features: 65.6 s&#10;Datasets: 71&#10;Next to MI: 13.7%, 46.4 s' aria-label='mRMR&#10;Improvement over Random: 13.5%&#10;Selection time per 1k features: 65.6 s&#10;Datasets: 71&#10;Next to MI: 13.7%, 46.4 s'/><circle class='sa-hit' cx='695.4' cy='111.1' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 13.3%&#10;Selection time per 1k features: 2.12e+03 s&#10;Datasets: 71&#10;Next to Lasso: 13.7%, 1.99e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 13.3%&#10;Selection time per 1k features: 2.12e+03 s&#10;Datasets: 71&#10;Next to Lasso: 13.7%, 1.99e+03 s'/><circle class='sa-hit' cx='798.3' cy='120.0' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 12.8%&#10;Selection time per 1k features: 2.38e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 12.8%&#10;Selection time per 1k features: 2.38e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='477.6' cy='129.1' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 12.3%, 18.9 s' aria-label='GainRatio&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 12.3%, 18.9 s'/><circle class='sa-hit' cx='494.6' cy='130.1' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 12.3%, 12.7 s' aria-label='MarkovBlanket&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 12.3%, 12.7 s'/><circle class='sa-hit' cx='813.0' cy='220.6' r='10.5' data-tip='SFS&#10;Improvement over Random: 7.3%&#10;Selection time per 1k features: 3.36e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 7.3%&#10;Selection time per 1k features: 3.36e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='742.2' cy='240.7' r='10.5' data-tip='LOCO&#10;Improvement over Random: 6.2%&#10;Selection time per 1k features: 6.37e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 6.2%&#10;Selection time per 1k features: 6.37e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='799.0' cy='247.0' r='10.5' data-tip='SBE&#10;Improvement over Random: 5.8%&#10;Selection time per 1k features: 2.42e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 5.8%&#10;Selection time per 1k features: 2.42e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='116.1' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,17.68,38.5146,71,True
37
  CART,14.296,14.4215,71,True
38
  F-Test,14.1513,0.6455,71,True
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='173.5' x2='173.5' y1='16.2' y2='368.0'/><text class='sa-text' x='173.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='271.4' x2='271.4' y1='16.2' y2='368.0'/><text class='sa-text' x='271.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='369.4' x2='369.4' y1='16.2' y2='368.0'/><text class='sa-text' x='369.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='467.4' x2='467.4' y1='16.2' y2='368.0'/><text class='sa-text' x='467.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='565.4' x2='565.4' y1='16.2' y2='368.0'/><text class='sa-text' x='565.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.4' x2='663.4' y1='16.2' y2='368.0'/><text class='sa-text' x='663.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.4' x2='761.4' y1='16.2' y2='368.0'/><text class='sa-text' x='761.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='81.3' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='72.3' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='81.3' x2='847.8' y1='306.8' y2='306.8'/><text class='sa-text' x='72.3' y='312.8' text-anchor='end' font-size='15'>2.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='261.6' y2='261.6'/><text class='sa-text' x='72.3' y='267.6' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='216.4' y2='216.4'/><text class='sa-text' x='72.3' y='222.4' text-anchor='end' font-size='15'>7.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='171.1' y2='171.1'/><text class='sa-text' x='72.3' y='177.1' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='81.3' x2='847.8' y1='125.9' y2='125.9'/><text class='sa-text' x='72.3' y='131.9' text-anchor='end' font-size='15'>12.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='80.7' y2='80.7'/><text class='sa-text' x='72.3' y='86.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='81.3' x2='847.8' y1='35.4' y2='35.4'/><text class='sa-text' x='72.3' y='41.4' text-anchor='end' font-size='15'>17.5</text><rect class='sa-frame-line' x='81.3' y='16.2' width='766.5' height='351.8'/><text class='sa-text' x='464.5' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='123.6' y1='51.9' x2='83.3' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='96.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='116.1,352.0 350.8,96.0 483.0,93.4 524.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='524.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='483.0' cy='93.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='350.8' cy='96.0' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='532.7' cy='103.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='692.7' cy='104.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='547.5' cy='108.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='111.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='798.3' cy='120.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='477.6' cy='129.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='494.6' cy='130.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='813.0' cy='220.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='742.2' cy='240.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='799.0' cy='247.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='116.1' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='524.8' y1='32.2' x2='647.2' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='647.2' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-text sa-halo' x='499.7' y='82.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='350.8' y1='96.0' x2='302.0' y2='100.0' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='302.0' y='100.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='532.7' y1='103.7' x2='570.4' y2='53.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='570.4' y='53.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='692.7' y1='104.6' x2='715.7' y2='89.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='715.7' y='89.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='547.5' y1='108.5' x2='606.8' y2='100.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='606.8' y='100.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='695.4' y1='111.1' x2='720.3' y2='131.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='720.3' y='131.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='798.3' y1='120.0' x2='797.9' y2='160.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='797.9' y='160.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='477.6' y1='129.1' x2='404.0' y2='142.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='404.0' y='142.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='494.6' y1='130.1' x2='579.9' y2='172.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='579.9' y='172.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='813.0' y1='220.6' x2='828.9' y2='206.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.9' y='206.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='742.2' y1='240.7' x2='708.8' y2='255.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='708.8' y='255.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='799.0' y1='247.0' x2='825.0' y2='260.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='825.0' y='260.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='116.1' y1='352.0' x2='168.9' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='168.9' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='524.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='483.0' cy='93.4' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 14.3%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 14.3%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71'/><circle class='sa-hit' cx='350.8' cy='96.0' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 14.2%&#10;Selection time per 1k features: 0.645 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 14.2%&#10;Selection time per 1k features: 0.645 s&#10;Datasets: 71'/><circle class='sa-hit' cx='532.7' cy='103.7' r='10.5' data-tip='MI&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 46.4 s&#10;Datasets: 71&#10;Next to mRMR: 13.5%, 65.6 s' aria-label='MI&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 46.4 s&#10;Datasets: 71&#10;Next to mRMR: 13.5%, 65.6 s'/><circle class='sa-hit' cx='692.7' cy='104.6' r='10.5' data-tip='Lasso&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.3%, 2.12e+03 s' aria-label='Lasso&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.3%, 2.12e+03 s'/><circle class='sa-hit' cx='547.5' cy='108.5' r='10.5' data-tip='mRMR&#10;Improvement over Random: 13.5%&#10;Selection time per 1k features: 65.6 s&#10;Datasets: 71&#10;Next to MI: 13.7%, 46.4 s' aria-label='mRMR&#10;Improvement over Random: 13.5%&#10;Selection time per 1k features: 65.6 s&#10;Datasets: 71&#10;Next to MI: 13.7%, 46.4 s'/><circle class='sa-hit' cx='695.4' cy='111.1' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 13.3%&#10;Selection time per 1k features: 2.12e+03 s&#10;Datasets: 71&#10;Next to Lasso: 13.7%, 1.99e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 13.3%&#10;Selection time per 1k features: 2.12e+03 s&#10;Datasets: 71&#10;Next to Lasso: 13.7%, 1.99e+03 s'/><circle class='sa-hit' cx='798.3' cy='120.0' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 12.8%&#10;Selection time per 1k features: 2.38e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 12.8%&#10;Selection time per 1k features: 2.38e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='477.6' cy='129.1' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 12.3%, 18.9 s' aria-label='GainRatio&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 12.3%, 18.9 s'/><circle class='sa-hit' cx='494.6' cy='130.1' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 12.3%, 12.7 s' aria-label='MarkovBlanket&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 12.3%, 12.7 s'/><circle class='sa-hit' cx='813.0' cy='220.6' r='10.5' data-tip='SFS&#10;Improvement over Random: 7.3%&#10;Selection time per 1k features: 3.36e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 7.3%&#10;Selection time per 1k features: 3.36e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='742.2' cy='240.7' r='10.5' data-tip='LOCO&#10;Improvement over Random: 6.2%&#10;Selection time per 1k features: 6.37e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 6.2%&#10;Selection time per 1k features: 6.37e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='799.0' cy='247.0' r='10.5' data-tip='SBE&#10;Improvement over Random: 5.8%&#10;Selection time per 1k features: 2.42e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 5.8%&#10;Selection time per 1k features: 2.42e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='116.1' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,17.68,38.5146,71,True
37
  CART,14.296,14.4215,71,True
38
  F-Test,14.1513,0.6455,71,True
data/subsets/tasks_all/models_lightgbm/budgets_4/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+52 / −44</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1177.0'>1177<span class='sub'>+57 / −49</span></td><td class='num' data-sort='7.7'>7.7</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+49 / −49</span></td><td class='num' data-sort='6.3'>6.3</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+46 / −41</span></td><td class='num' data-sort='7.6'>7.6</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1160.0'>1160<span class='sub'>+44 / −41</span></td><td class='num' data-sort='4.6'>4.6</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1154.0'>1154<span class='sub'>+45 / −50</span></td><td class='num' data-sort='9.2'>9.2</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1148.0'>1148<span class='sub'>+45 / −48</span></td><td class='num' data-sort='7.0'>7.0</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1125.0'>1125<span class='sub'>+45 / −48</span></td><td class='num' data-sort='8.3'>8.3</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1116.0'>1116<span class='sub'>+46 / −48</span></td><td class='num' data-sort='7.1'>7.1</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1112.0'>1112<span class='sub'>+51 / −53</span></td><td class='num' data-sort='5.3'>5.3</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='1001.8'>1,001.8</td><td class='num' data-sort='40.8'>40.8</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+50 / −49</span></td><td class='num' data-sort='4.0'>4.0</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='3227.2'>3,227.2</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1092.0'>1092<span class='sub'>+53 / −50</span></td><td class='num' data-sort='1.2'>1.2</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='249.4'>249.4</td><td class='num' data-sort='18.3'>18.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1091.0'>1091<span class='sub'>+41 / −46</span></td><td class='num' data-sort='5.1'>5.1</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+53 / −56</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='968.0'>968<span class='sub'>+56 / −74</span></td><td class='num' data-sort='-220.2'>-220.2</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1274.0,1230.0,1326.0,11.1,0.597,0.507,0.2,0.0
37
  2,CART,Non-search,Tree,1177.0,1128.0,1234.0,7.7,0.324,0.482,0.1,0.0
38
  3,ElasticNet,Non-search,Regularization,1171.0,1122.0,1220.0,6.3,0.308,0.568,6.1,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+52 / −44</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1177.0'>1177<span class='sub'>+57 / −49</span></td><td class='num' data-sort='7.7'>7.7</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+49 / −49</span></td><td class='num' data-sort='6.3'>6.3</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+46 / −41</span></td><td class='num' data-sort='7.6'>7.6</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1160.0'>1160<span class='sub'>+44 / −41</span></td><td class='num' data-sort='4.6'>4.6</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1154.0'>1154<span class='sub'>+45 / −50</span></td><td class='num' data-sort='9.2'>9.2</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1148.0'>1148<span class='sub'>+45 / −48</span></td><td class='num' data-sort='7.0'>7.0</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1125.0'>1125<span class='sub'>+45 / −48</span></td><td class='num' data-sort='8.3'>8.3</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1116.0'>1116<span class='sub'>+46 / −48</span></td><td class='num' data-sort='7.1'>7.1</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1112.0'>1112<span class='sub'>+51 / −53</span></td><td class='num' data-sort='5.3'>5.3</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='1001.8'>1,001.8</td><td class='num' data-sort='40.8'>40.8</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+50 / −49</span></td><td class='num' data-sort='4.0'>4.0</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='3227.2'>3,227.2</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1092.0'>1092<span class='sub'>+53 / −50</span></td><td class='num' data-sort='1.2'>1.2</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='249.4'>249.4</td><td class='num' data-sort='18.3'>18.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1091.0'>1091<span class='sub'>+41 / −46</span></td><td class='num' data-sort='5.1'>5.1</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+53 / −56</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='968.0'>968<span class='sub'>+56 / −74</span></td><td class='num' data-sort='-220.2'>-220.2</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1274.0,1230.0,1326.0,11.1,0.597,0.507,0.2,0.0
39
  2,CART,Non-search,Tree,1177.0,1128.0,1234.0,7.7,0.324,0.482,0.1,0.0
40
  3,ElasticNet,Non-search,Regularization,1171.0,1122.0,1220.0,6.3,0.308,0.568,6.1,0.0
data/subsets/tasks_all/models_lightgbm/budgets_4/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='159.1' x2='159.1' y1='16.2' y2='368.0'/><text class='sa-text' x='159.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='258.8' x2='258.8' y1='16.2' y2='368.0'/><text class='sa-text' x='258.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='358.5' x2='358.5' y1='16.2' y2='368.0'/><text class='sa-text' x='358.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='458.2' x2='458.2' y1='16.2' y2='368.0'/><text class='sa-text' x='458.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='558.0' x2='558.0' y1='16.2' y2='368.0'/><text class='sa-text' x='558.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.7' x2='657.7' y1='16.2' y2='368.0'/><text class='sa-text' x='657.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.4' x2='757.4' y1='16.2' y2='368.0'/><text class='sa-text' x='757.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='294.4' y2='294.4'/><text class='sa-text' x='57.9' y='300.4' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='236.8' y2='236.8'/><text class='sa-text' x='57.9' y='242.8' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='179.1' y2='179.1'/><text class='sa-text' x='57.9' y='185.1' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='121.5' y2='121.5'/><text class='sa-text' x='57.9' y='127.5' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='63.8' y2='63.8'/><text class='sa-text' x='57.9' y='69.8' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 338.2,88.2 516.9,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.9' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='338.2' cy='88.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='563.3' cy='114.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.4' cy='129.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='688.9' cy='134.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='483.5' cy='146.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.1' cy='149.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='689.0' cy='169.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='776.9' cy='199.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.0' cy='206.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='524.8' cy='219.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='237.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.3' cy='318.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.9' y1='32.2' x2='410.5' y2='46.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='410.5' y='46.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='338.2' y1='88.2' x2='298.6' y2='102.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='298.6' y='102.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='563.3' y1='114.1' x2='608.6' y2='123.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='608.6' y='123.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='474.4' y1='129.0' x2='495.8' y2='114.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='495.8' y='114.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='688.9' y1='134.3' x2='710.3' y2='104.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='710.3' y='104.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='483.5' y1='146.4' x2='590.0' y2='158.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='590.0' y='158.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><text class='sa-dim sa-halo' x='796.9' y='136.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><text class='sa-dim sa-halo' x='697.2' y='188.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='776.9' y1='199.6' x2='817.2' y2='196.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='817.2' y='196.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><text class='sa-dim sa-halo' x='458.6' y='191.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='543.1' y='220.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='812.3' y1='237.0' x2='828.6' y2='251.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='251.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='737.3' y1='318.6' x2='703.3' y2='332.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='703.3' y='332.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.9' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71'/><circle class='sa-hit' cx='338.2' cy='88.2' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 0.626 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 0.626 s&#10;Datasets: 71'/><circle class='sa-hit' cx='563.3' cy='114.1' r='10.5' data-tip='mRMR&#10;Improvement over Random: 8.3%&#10;Selection time per 1k features: 113 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 8.3%&#10;Selection time per 1k features: 113 s&#10;Datasets: 71'/><circle class='sa-hit' cx='474.4' cy='129.0' r='10.5' data-tip='CART&#10;Improvement over Random: 7.7%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 7.1%, 17.9 s' aria-label='CART&#10;Improvement over Random: 7.7%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 7.1%, 17.9 s'/><circle class='sa-hit' cx='688.9' cy='134.3' r='10.5' data-tip='Lasso&#10;Improvement over Random: 7.6%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 71' aria-label='Lasso&#10;Improvement over Random: 7.6%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='483.5' cy='146.4' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 7.1%&#10;Selection time per 1k features: 17.9 s&#10;Datasets: 71&#10;Next to CART: 7.7%, 14.5 s' aria-label='MarkovBlanket&#10;Improvement over Random: 7.1%&#10;Selection time per 1k features: 17.9 s&#10;Datasets: 71&#10;Next to CART: 7.7%, 14.5 s'/><circle class='sa-hit' cx='795.1' cy='149.4' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 7.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 7.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='689.0' cy='169.3' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 6.3%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 71' aria-label='ElasticNet&#10;Improvement over Random: 6.3%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='776.9' cy='199.6' r='10.5' data-tip='SBE&#10;Improvement over Random: 5.3%&#10;Selection time per 1k features: 1.57e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 5.3%&#10;Selection time per 1k features: 1.57e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='468.0' cy='206.5' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 5.1%&#10;Selection time per 1k features: 12.5 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 5.1%&#10;Selection time per 1k features: 12.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='524.8' cy='219.1' r='10.5' data-tip='MI&#10;Improvement over Random: 4.6%&#10;Selection time per 1k features: 46.5 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 4.6%&#10;Selection time per 1k features: 46.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='237.0' r='10.5' data-tip='SFS&#10;Improvement over Random: 4.0%&#10;Selection time per 1k features: 3.56e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 4.0%&#10;Selection time per 1k features: 3.56e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='737.3' cy='318.6' r='10.5' data-tip='LOCO&#10;Improvement over Random: 1.2%&#10;Selection time per 1k features: 6.29e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 1.2%&#10;Selection time per 1k features: 6.29e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,11.0971,38.79,71,True
37
  F-Test,9.1536,0.6257,71,True
38
  mRMR,8.2563,113.024,71,False
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='159.1' x2='159.1' y1='16.2' y2='368.0'/><text class='sa-text' x='159.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='258.8' x2='258.8' y1='16.2' y2='368.0'/><text class='sa-text' x='258.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='358.5' x2='358.5' y1='16.2' y2='368.0'/><text class='sa-text' x='358.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='458.2' x2='458.2' y1='16.2' y2='368.0'/><text class='sa-text' x='458.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='558.0' x2='558.0' y1='16.2' y2='368.0'/><text class='sa-text' x='558.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.7' x2='657.7' y1='16.2' y2='368.0'/><text class='sa-text' x='657.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.4' x2='757.4' y1='16.2' y2='368.0'/><text class='sa-text' x='757.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='294.4' y2='294.4'/><text class='sa-text' x='57.9' y='300.4' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='236.8' y2='236.8'/><text class='sa-text' x='57.9' y='242.8' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='179.1' y2='179.1'/><text class='sa-text' x='57.9' y='185.1' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='121.5' y2='121.5'/><text class='sa-text' x='57.9' y='127.5' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='63.8' y2='63.8'/><text class='sa-text' x='57.9' y='69.8' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 338.2,88.2 516.9,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.9' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='338.2' cy='88.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='563.3' cy='114.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.4' cy='129.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='688.9' cy='134.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='483.5' cy='146.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.1' cy='149.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='689.0' cy='169.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='776.9' cy='199.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.0' cy='206.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='524.8' cy='219.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='237.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.3' cy='318.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.9' y1='32.2' x2='413.0' y2='45.9' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='413.0' y='45.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='338.2' y1='88.2' x2='299.8' y2='101.9' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='299.8' y='101.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='563.3' y1='114.1' x2='608.6' y2='124.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='608.6' y='124.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='474.4' y1='129.0' x2='495.8' y2='114.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='495.8' y='114.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='688.9' y1='134.3' x2='712.8' y2='106.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='712.8' y='106.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='483.5' y1='146.4' x2='594.4' y2='152.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='594.4' y='152.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><text class='sa-dim sa-halo' x='796.9' y='135.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='689.0' y1='169.3' x2='699.8' y2='188.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='699.8' y='188.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='776.9' y1='199.6' x2='815.3' y2='196.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='815.3' y='196.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='468.0' y1='206.5' x2='451.0' y2='191.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='451.0' y='191.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='545.0' y='224.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='812.3' y1='237.0' x2='828.6' y2='250.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='250.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='737.3' y1='318.6' x2='704.6' y2='332.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='704.6' y='332.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.9' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71'/><circle class='sa-hit' cx='338.2' cy='88.2' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 0.626 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 0.626 s&#10;Datasets: 71'/><circle class='sa-hit' cx='563.3' cy='114.1' r='10.5' data-tip='mRMR&#10;Improvement over Random: 8.3%&#10;Selection time per 1k features: 113 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 8.3%&#10;Selection time per 1k features: 113 s&#10;Datasets: 71'/><circle class='sa-hit' cx='474.4' cy='129.0' r='10.5' data-tip='CART&#10;Improvement over Random: 7.7%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 7.1%, 17.9 s' aria-label='CART&#10;Improvement over Random: 7.7%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 7.1%, 17.9 s'/><circle class='sa-hit' cx='688.9' cy='134.3' r='10.5' data-tip='Lasso&#10;Improvement over Random: 7.6%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 71' aria-label='Lasso&#10;Improvement over Random: 7.6%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='483.5' cy='146.4' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 7.1%&#10;Selection time per 1k features: 17.9 s&#10;Datasets: 71&#10;Next to CART: 7.7%, 14.5 s' aria-label='MarkovBlanket&#10;Improvement over Random: 7.1%&#10;Selection time per 1k features: 17.9 s&#10;Datasets: 71&#10;Next to CART: 7.7%, 14.5 s'/><circle class='sa-hit' cx='795.1' cy='149.4' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 7.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 7.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='689.0' cy='169.3' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 6.3%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 71' aria-label='ElasticNet&#10;Improvement over Random: 6.3%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='776.9' cy='199.6' r='10.5' data-tip='SBE&#10;Improvement over Random: 5.3%&#10;Selection time per 1k features: 1.57e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 5.3%&#10;Selection time per 1k features: 1.57e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='468.0' cy='206.5' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 5.1%&#10;Selection time per 1k features: 12.5 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 5.1%&#10;Selection time per 1k features: 12.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='524.8' cy='219.1' r='10.5' data-tip='MI&#10;Improvement over Random: 4.6%&#10;Selection time per 1k features: 46.5 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 4.6%&#10;Selection time per 1k features: 46.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='237.0' r='10.5' data-tip='SFS&#10;Improvement over Random: 4.0%&#10;Selection time per 1k features: 3.56e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 4.0%&#10;Selection time per 1k features: 3.56e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='737.3' cy='318.6' r='10.5' data-tip='LOCO&#10;Improvement over Random: 1.2%&#10;Selection time per 1k features: 6.29e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 1.2%&#10;Selection time per 1k features: 6.29e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,11.0971,38.79,71,True
37
  F-Test,9.1536,0.6257,71,True
38
  mRMR,8.2563,113.024,71,False
data/subsets/tasks_all/models_lightgbm/budgets_all/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1313.0'>1313<span class='sub'>+44 / −39</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1267.0'>1267<span class='sub'>+39 / −33</span></td><td class='num' data-sort='23.5'>23.5</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+36 / −37</span></td><td class='num' data-sort='21.6'>21.6</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1244.0'>1244<span class='sub'>+32 / −30</span></td><td class='num' data-sort='22.9'>22.9</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1244.0'>1244<span class='sub'>+36 / −36</span></td><td class='num' data-sort='21.1'>21.1</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1224.0'>1224<span class='sub'>+31 / −28</span></td><td class='num' data-sort='21.2'>21.2</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1217.0'>1217<span class='sub'>+34 / −34</span></td><td class='num' data-sort='21.5'>21.5</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1201.0'>1201<span class='sub'>+33 / −33</span></td><td class='num' data-sort='21.7'>21.7</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+38 / −40</span></td><td class='num' data-sort='18.1'>18.1</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2166.9'>2,166.9</td><td class='num' data-sort='43.8'>43.8</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1185.0'>1185<span class='sub'>+37 / −36</span></td><td class='num' data-sort='18.5'>18.5</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='13.1'>13.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1184.0'>1184<span class='sub'>+44 / −46</span></td><td class='num' data-sort='12.3'>12.3</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2611.6'>2,611.6</td><td class='num' data-sort='46.2'>46.2</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1181.0'>1181<span class='sub'>+32 / −33</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1158.0'>1158<span class='sub'>+41 / −40</span></td><td class='num' data-sort='13.0'>13.0</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='240.9'>240.9</td><td class='num' data-sort='7.2'>7.2</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+36 / −38</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='943.0'>943<span class='sub'>+54 / −65</span></td><td class='num' data-sort='-23.2'>-23.2</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1313.0,1274.0,1357.0,25.5,0.643,0.669,0.2,0.0
37
  2,CART,Non-search,Tree,1267.0,1234.0,1306.0,23.5,0.422,0.586,0.0,0.0
38
  3,ElasticNet,Non-search,Regularization,1252.0,1215.0,1288.0,21.6,0.447,0.76,6.1,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1313.0'>1313<span class='sub'>+44 / −39</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1267.0'>1267<span class='sub'>+39 / −33</span></td><td class='num' data-sort='23.5'>23.5</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+36 / −37</span></td><td class='num' data-sort='21.6'>21.6</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1244.0'>1244<span class='sub'>+32 / −30</span></td><td class='num' data-sort='22.9'>22.9</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1244.0'>1244<span class='sub'>+36 / −36</span></td><td class='num' data-sort='21.1'>21.1</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1224.0'>1224<span class='sub'>+31 / −28</span></td><td class='num' data-sort='21.2'>21.2</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1217.0'>1217<span class='sub'>+34 / −34</span></td><td class='num' data-sort='21.5'>21.5</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1201.0'>1201<span class='sub'>+33 / −33</span></td><td class='num' data-sort='21.7'>21.7</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+38 / −40</span></td><td class='num' data-sort='18.1'>18.1</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2166.9'>2,166.9</td><td class='num' data-sort='43.8'>43.8</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1185.0'>1185<span class='sub'>+37 / −36</span></td><td class='num' data-sort='18.5'>18.5</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='13.1'>13.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1184.0'>1184<span class='sub'>+44 / −46</span></td><td class='num' data-sort='12.3'>12.3</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2611.6'>2,611.6</td><td class='num' data-sort='46.2'>46.2</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1181.0'>1181<span class='sub'>+32 / −33</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1158.0'>1158<span class='sub'>+41 / −40</span></td><td class='num' data-sort='13.0'>13.0</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='240.9'>240.9</td><td class='num' data-sort='7.2'>7.2</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+36 / −38</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='943.0'>943<span class='sub'>+54 / −65</span></td><td class='num' data-sort='-23.2'>-23.2</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1313.0,1274.0,1357.0,25.5,0.643,0.669,0.2,0.0
39
  2,CART,Non-search,Tree,1267.0,1234.0,1306.0,23.5,0.422,0.586,0.0,0.0
40
  3,ElasticNet,Non-search,Regularization,1252.0,1215.0,1288.0,21.6,0.447,0.76,6.1,0.0
data/subsets/tasks_all/models_lightgbm/budgets_all/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='161.4' x2='161.4' y1='16.2' y2='368.0'/><text class='sa-text' x='161.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='262.2' x2='262.2' y1='16.2' y2='368.0'/><text class='sa-text' x='262.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.9' x2='362.9' y1='16.2' y2='368.0'/><text class='sa-text' x='362.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='463.7' x2='463.7' y1='16.2' y2='368.0'/><text class='sa-text' x='463.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='564.5' x2='564.5' y1='16.2' y2='368.0'/><text class='sa-text' x='564.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='665.3' x2='665.3' y1='16.2' y2='368.0'/><text class='sa-text' x='665.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='766.1' x2='766.1' y1='16.2' y2='368.0'/><text class='sa-text' x='766.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='289.4' y2='289.4'/><text class='sa-text' x='57.9' y='295.4' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='226.8' y2='226.8'/><text class='sa-text' x='57.9' y='232.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='164.2' y2='164.2'/><text class='sa-text' x='57.9' y='170.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='101.6' y2='101.6'/><text class='sa-text' x='57.9' y='107.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='39.0' y2='39.0'/><text class='sa-text' x='57.9' y='45.0' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 343.0,83.2 480.0,57.7 522.9,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='522.9' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='480.0' cy='57.7' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='530.8' cy='65.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='534.0' cy='81.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='698.0' cy='81.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='343.0' cy='83.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='491.2' cy='86.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='88.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.2' cy='104.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='804.2' cy='120.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='125.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='745.9' cy='189.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='810.3' cy='197.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='522.9' y1='32.2' x2='647.6' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='647.6' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='480.0' y1='57.7' x2='440.9' y2='71.4' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='440.9' y='71.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><text class='sa-dim sa-halo' x='549.0' y='54.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='534.0' y1='81.0' x2='584.4' y2='91.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='584.4' y='91.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='698.0' y1='81.7' x2='793.2' y2='71.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='793.2' y='71.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='343.0' y1='83.2' x2='301.5' y2='97.4' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='301.5' y='97.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='491.2' y1='86.9' x2='575.0' y2='125.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='575.0' y='125.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='695.4' y1='88.3' x2='665.5' y2='55.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='665.5' y='55.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='474.2' y1='104.5' x2='454.7' y2='153.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='454.7' y='153.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='804.2' y1='120.6' x2='786.9' y2='105.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='786.9' y='105.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='125.7' x2='828.6' y2='140.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='140.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='745.9' y1='189.2' x2='769.2' y2='174.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='769.2' y='174.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='810.3' y1='197.4' x2='827.3' y2='211.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='827.3' y='211.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='522.9' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 38.6 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 38.6 s&#10;Datasets: 71'/><circle class='sa-hit' cx='480.0' cy='57.7' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 23.5%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 23.5%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='530.8' cy='65.1' r='10.5' data-tip='MI&#10;Improvement over Random: 22.9%&#10;Selection time per 1k features: 46.3 s&#10;Datasets: 71&#10;Next to mRMR: 21.7%, 49.9 s' aria-label='MI&#10;Improvement over Random: 22.9%&#10;Selection time per 1k features: 46.3 s&#10;Datasets: 71&#10;Next to mRMR: 21.7%, 49.9 s'/><circle class='sa-hit' cx='534.0' cy='81.0' r='10.5' data-tip='mRMR&#10;Improvement over Random: 21.7%&#10;Selection time per 1k features: 49.9 s&#10;Datasets: 71&#10;Next to MI: 22.9%, 46.3 s' aria-label='mRMR&#10;Improvement over Random: 21.7%&#10;Selection time per 1k features: 49.9 s&#10;Datasets: 71&#10;Next to MI: 22.9%, 46.3 s'/><circle class='sa-hit' cx='698.0' cy='81.7' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 21.6%&#10;Selection time per 1k features: 2.11e+03 s&#10;Datasets: 71&#10;Next to Lasso: 21.1%, 1.99e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 21.6%&#10;Selection time per 1k features: 2.11e+03 s&#10;Datasets: 71&#10;Next to Lasso: 21.1%, 1.99e+03 s'/><circle class='sa-hit' cx='343.0' cy='83.2' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 21.5%&#10;Selection time per 1k features: 0.635 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 21.5%&#10;Selection time per 1k features: 0.635 s&#10;Datasets: 71'/><circle class='sa-hit' cx='491.2' cy='86.9' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 21.2%&#10;Selection time per 1k features: 18.7 s&#10;Datasets: 71' aria-label='MarkovBlanket&#10;Improvement over Random: 21.2%&#10;Selection time per 1k features: 18.7 s&#10;Datasets: 71'/><circle class='sa-hit' cx='695.4' cy='88.3' r='10.5' data-tip='Lasso&#10;Improvement over Random: 21.1%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 21.6%, 2.11e+03 s' aria-label='Lasso&#10;Improvement over Random: 21.1%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 21.6%, 2.11e+03 s'/><circle class='sa-hit' cx='474.2' cy='104.5' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71'/><circle class='sa-hit' cx='804.2' cy='120.6' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 18.5%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71&#10;Next to SFS: 18.1%, 2.88e+04 s' aria-label='(R)ReliefF&#10;Improvement over Random: 18.5%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71&#10;Next to SFS: 18.1%, 2.88e+04 s'/><circle class='sa-hit' cx='812.3' cy='125.7' r='10.5' data-tip='SFS&#10;Improvement over Random: 18.1%&#10;Selection time per 1k features: 2.88e+04 s&#10;Datasets: 71&#10;Next to (R)ReliefF: 18.5%, 2.39e+04 s' aria-label='SFS&#10;Improvement over Random: 18.1%&#10;Selection time per 1k features: 2.88e+04 s&#10;Datasets: 71&#10;Next to (R)ReliefF: 18.5%, 2.39e+04 s'/><circle class='sa-hit' cx='745.9' cy='189.2' r='10.5' data-tip='LOCO&#10;Improvement over Random: 13.0%&#10;Selection time per 1k features: 6.31e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 13.0%&#10;Selection time per 1k features: 6.31e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='810.3' cy='197.4' r='10.5' data-tip='SBE&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 2.75e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 2.75e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,25.5467,38.6316,71,True
37
  CART,23.5071,14.4928,71,True
38
  MI,22.9183,46.2611,71,False
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='161.4' x2='161.4' y1='16.2' y2='368.0'/><text class='sa-text' x='161.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='262.2' x2='262.2' y1='16.2' y2='368.0'/><text class='sa-text' x='262.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.9' x2='362.9' y1='16.2' y2='368.0'/><text class='sa-text' x='362.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='463.7' x2='463.7' y1='16.2' y2='368.0'/><text class='sa-text' x='463.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='564.5' x2='564.5' y1='16.2' y2='368.0'/><text class='sa-text' x='564.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='665.3' x2='665.3' y1='16.2' y2='368.0'/><text class='sa-text' x='665.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='766.1' x2='766.1' y1='16.2' y2='368.0'/><text class='sa-text' x='766.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='289.4' y2='289.4'/><text class='sa-text' x='57.9' y='295.4' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='226.8' y2='226.8'/><text class='sa-text' x='57.9' y='232.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='164.2' y2='164.2'/><text class='sa-text' x='57.9' y='170.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='101.6' y2='101.6'/><text class='sa-text' x='57.9' y='107.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='39.0' y2='39.0'/><text class='sa-text' x='57.9' y='45.0' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 343.0,83.2 480.0,57.7 522.9,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='522.9' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='480.0' cy='57.7' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='530.8' cy='65.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='534.0' cy='81.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='698.0' cy='81.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='343.0' cy='83.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='491.2' cy='86.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='88.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.2' cy='104.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='804.2' cy='120.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='125.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='745.9' cy='189.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='810.3' cy='197.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='522.9' y1='32.2' x2='647.6' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='647.6' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='480.0' y1='57.7' x2='440.9' y2='71.4' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='440.9' y='71.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><text class='sa-dim sa-halo' x='549.0' y='54.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='534.0' y1='81.0' x2='586.9' y2='86.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='586.9' y='86.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='698.0' y1='81.7' x2='790.7' y2='74.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='790.7' y='74.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='343.0' y1='83.2' x2='301.5' y2='97.4' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='301.5' y='97.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='491.2' y1='86.9' x2='582.5' y2='122.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='582.5' y='122.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='695.4' y1='88.3' x2='666.8' y2='59.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='666.8' y='59.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='474.2' y1='104.5' x2='454.7' y2='153.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='454.7' y='153.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='796.9' y='105.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='125.7' x2='828.6' y2='140.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='140.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='745.9' y1='189.2' x2='769.2' y2='174.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='769.2' y='174.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='810.3' y1='197.4' x2='827.3' y2='211.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='827.3' y='211.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='522.9' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 38.6 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 38.6 s&#10;Datasets: 71'/><circle class='sa-hit' cx='480.0' cy='57.7' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 23.5%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 23.5%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='530.8' cy='65.1' r='10.5' data-tip='MI&#10;Improvement over Random: 22.9%&#10;Selection time per 1k features: 46.3 s&#10;Datasets: 71&#10;Next to mRMR: 21.7%, 49.9 s' aria-label='MI&#10;Improvement over Random: 22.9%&#10;Selection time per 1k features: 46.3 s&#10;Datasets: 71&#10;Next to mRMR: 21.7%, 49.9 s'/><circle class='sa-hit' cx='534.0' cy='81.0' r='10.5' data-tip='mRMR&#10;Improvement over Random: 21.7%&#10;Selection time per 1k features: 49.9 s&#10;Datasets: 71&#10;Next to MI: 22.9%, 46.3 s' aria-label='mRMR&#10;Improvement over Random: 21.7%&#10;Selection time per 1k features: 49.9 s&#10;Datasets: 71&#10;Next to MI: 22.9%, 46.3 s'/><circle class='sa-hit' cx='698.0' cy='81.7' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 21.6%&#10;Selection time per 1k features: 2.11e+03 s&#10;Datasets: 71&#10;Next to Lasso: 21.1%, 1.99e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 21.6%&#10;Selection time per 1k features: 2.11e+03 s&#10;Datasets: 71&#10;Next to Lasso: 21.1%, 1.99e+03 s'/><circle class='sa-hit' cx='343.0' cy='83.2' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 21.5%&#10;Selection time per 1k features: 0.635 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 21.5%&#10;Selection time per 1k features: 0.635 s&#10;Datasets: 71'/><circle class='sa-hit' cx='491.2' cy='86.9' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 21.2%&#10;Selection time per 1k features: 18.7 s&#10;Datasets: 71' aria-label='MarkovBlanket&#10;Improvement over Random: 21.2%&#10;Selection time per 1k features: 18.7 s&#10;Datasets: 71'/><circle class='sa-hit' cx='695.4' cy='88.3' r='10.5' data-tip='Lasso&#10;Improvement over Random: 21.1%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 21.6%, 2.11e+03 s' aria-label='Lasso&#10;Improvement over Random: 21.1%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 21.6%, 2.11e+03 s'/><circle class='sa-hit' cx='474.2' cy='104.5' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71'/><circle class='sa-hit' cx='804.2' cy='120.6' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 18.5%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71&#10;Next to SFS: 18.1%, 2.88e+04 s' aria-label='(R)ReliefF&#10;Improvement over Random: 18.5%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71&#10;Next to SFS: 18.1%, 2.88e+04 s'/><circle class='sa-hit' cx='812.3' cy='125.7' r='10.5' data-tip='SFS&#10;Improvement over Random: 18.1%&#10;Selection time per 1k features: 2.88e+04 s&#10;Datasets: 71&#10;Next to (R)ReliefF: 18.5%, 2.39e+04 s' aria-label='SFS&#10;Improvement over Random: 18.1%&#10;Selection time per 1k features: 2.88e+04 s&#10;Datasets: 71&#10;Next to (R)ReliefF: 18.5%, 2.39e+04 s'/><circle class='sa-hit' cx='745.9' cy='189.2' r='10.5' data-tip='LOCO&#10;Improvement over Random: 13.0%&#10;Selection time per 1k features: 6.31e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 13.0%&#10;Selection time per 1k features: 6.31e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='810.3' cy='197.4' r='10.5' data-tip='SBE&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 2.75e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 12.3%&#10;Selection time per 1k features: 2.75e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,25.5467,38.6316,71,True
37
  CART,23.5071,14.4928,71,True
38
  MI,22.9183,46.2611,71,False
data/subsets/tasks_all/models_linearmodel/budgets_0/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1352.0'>1352<span class='sub'>+61 / −49</span></td><td class='num' data-sort='28.3'>28.3</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1350.0'>1350<span class='sub'>+48 / −44</span></td><td class='num' data-sort='28.8'>28.8</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1344.0'>1344<span class='sub'>+62 / −56</span></td><td class='num' data-sort='27.0'>27.0</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1327.0'>1327<span class='sub'>+54 / −48</span></td><td class='num' data-sort='26.2'>26.2</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='6.0'>6.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1324.0'>1324<span class='sub'>+44 / −43</span></td><td class='num' data-sort='28.0'>28.0</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1323.0'>1323<span class='sub'>+44 / −41</span></td><td class='num' data-sort='25.4'>25.4</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1301.0'>1301<span class='sub'>+45 / −48</span></td><td class='num' data-sort='26.3'>26.3</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1295.0'>1295<span class='sub'>+51 / −52</span></td><td class='num' data-sort='23.3'>23.3</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='970.7'>970.7</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1291.0'>1291<span class='sub'>+49 / −47</span></td><td class='num' data-sort='24.5'>24.5</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+44 / −41</span></td><td class='num' data-sort='22.5'>22.5</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1226.0'>1226<span class='sub'>+56 / −57</span></td><td class='num' data-sort='18.0'>18.0</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='13.7'>13.7</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+45 / −45</span></td><td class='num' data-sort='15.3'>15.3</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='245.6'>245.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+51 / −57</span></td><td class='num' data-sort='8.8'>8.8</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3600.1'>3,600.1</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+54 / −62</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='973.0'>973<span class='sub'>+68 / −83</span></td><td class='num' data-sort='-12.4'>-12.4</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1352.0,1303.0,1413.0,28.3,0.628,0.805,0.2,0.0
37
  2,F-Test,Non-search,Correlation,1350.0,1306.0,1398.0,28.8,0.725,0.952,0.0,0.0
38
  3,CART,Non-search,Tree,1344.0,1288.0,1406.0,27.0,0.484,0.731,0.0,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1352.0'>1352<span class='sub'>+61 / −49</span></td><td class='num' data-sort='28.3'>28.3</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1350.0'>1350<span class='sub'>+48 / −44</span></td><td class='num' data-sort='28.8'>28.8</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1344.0'>1344<span class='sub'>+62 / −56</span></td><td class='num' data-sort='27.0'>27.0</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1327.0'>1327<span class='sub'>+54 / −48</span></td><td class='num' data-sort='26.2'>26.2</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='6.0'>6.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1324.0'>1324<span class='sub'>+44 / −43</span></td><td class='num' data-sort='28.0'>28.0</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1323.0'>1323<span class='sub'>+44 / −41</span></td><td class='num' data-sort='25.4'>25.4</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1301.0'>1301<span class='sub'>+45 / −48</span></td><td class='num' data-sort='26.3'>26.3</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1295.0'>1295<span class='sub'>+51 / −52</span></td><td class='num' data-sort='23.3'>23.3</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='970.7'>970.7</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1291.0'>1291<span class='sub'>+49 / −47</span></td><td class='num' data-sort='24.5'>24.5</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+44 / −41</span></td><td class='num' data-sort='22.5'>22.5</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1226.0'>1226<span class='sub'>+56 / −57</span></td><td class='num' data-sort='18.0'>18.0</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='13.7'>13.7</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+45 / −45</span></td><td class='num' data-sort='15.3'>15.3</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='245.6'>245.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+51 / −57</span></td><td class='num' data-sort='8.8'>8.8</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3600.1'>3,600.1</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+54 / −62</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='973.0'>973<span class='sub'>+68 / −83</span></td><td class='num' data-sort='-12.4'>-12.4</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1352.0,1303.0,1413.0,28.3,0.628,0.805,0.2,0.0
39
  2,F-Test,Non-search,Correlation,1350.0,1306.0,1398.0,28.8,0.725,0.952,0.0,0.0
40
  3,CART,Non-search,Tree,1344.0,1288.0,1406.0,27.0,0.484,0.731,0.0,0.0
data/subsets/tasks_all/models_linearmodel/budgets_0/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='162.3' x2='162.3' y1='20.0' y2='368.0'/><text class='sa-text' x='162.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='261.6' x2='261.6' y1='20.0' y2='368.0'/><text class='sa-text' x='261.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='360.9' x2='360.9' y1='20.0' y2='368.0'/><text class='sa-text' x='360.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='460.3' x2='460.3' y1='20.0' y2='368.0'/><text class='sa-text' x='460.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.6' x2='559.6' y1='20.0' y2='368.0'/><text class='sa-text' x='559.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='658.9' x2='658.9' y1='20.0' y2='368.0'/><text class='sa-text' x='658.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='758.3' x2='758.3' y1='20.0' y2='368.0'/><text class='sa-text' x='758.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.2' y2='352.2'/><text class='sa-text' x='58.0' y='358.2' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='297.2' y2='297.2'/><text class='sa-text' x='58.0' y='303.2' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='242.2' y2='242.2'/><text class='sa-text' x='58.0' y='248.2' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='187.2' y2='187.2'/><text class='sa-text' x='58.0' y='193.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='132.2' y2='132.2'/><text class='sa-text' x='58.0' y='138.2' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='77.2' y2='77.2'/><text class='sa-text' x='58.0' y='83.2' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='22.2' y2='22.2'/><text class='sa-text' x='58.0' y='28.2' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='20.0' width='780.8' height='348.1'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 194.0) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='55.3' x2='69.0' y2='22.0' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='79.3' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.2 341.0,35.8' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='341.0' cy='35.8' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='518.2' cy='41.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='526.6' cy='44.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='476.4' cy='55.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='481.0' cy='62.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.8' cy='64.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='487.9' cy='72.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='688.2' cy='82.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='782.3' cy='96.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='471.1' cy='104.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.8' cy='154.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='738.2' cy='184.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='255.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.2' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='341.0' y1='35.8' x2='258.5' y2='31.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='258.5' y='31.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='518.2' y1='41.0' x2='422.5' y2='29.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='422.5' y='29.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>RFImportance</text><line class='sa-ink' x1='526.6' y1='44.3' x2='425.9' y2='29.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='425.9' y='29.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='476.4' y1='55.5' x2='422.2' y2='62.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='422.2' y='62.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='481.0' y1='62.6' x2='402.2' y2='90.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='402.2' y='90.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='691.8' y1='64.5' x2='738.4' y2='49.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='738.4' y='49.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='487.9' y1='72.6' x2='579.8' y2='94.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='579.8' y='94.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='688.2' y1='82.6' x2='747.1' y2='113.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='747.1' y='113.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='782.3' y1='96.1' x2='814.1' y2='81.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='814.1' y='81.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='471.1' y1='104.4' x2='530.9' y2='129.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='530.9' y='129.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='797.0' y='143.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='738.2' y1='184.3' x2='705.5' y2='197.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='705.5' y='197.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='812.3' y1='255.6' x2='787.1' y2='269.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='787.1' y='269.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.2' x2='156.2' y2='338.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='338.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='341.0' cy='35.8' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 28.8%&#10;Selection time per 1k features: 0.629 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 28.8%&#10;Selection time per 1k features: 0.629 s&#10;Datasets: 71'/><circle class='sa-hit' cx='518.2' cy='41.0' r='10.5' data-tip='RFImportance&#10;Improvement over Random: 28.3%&#10;Selection time per 1k features: 38.3 s&#10;Datasets: 71&#10;Next to MI: 28.0%, 46.6 s' aria-label='RFImportance&#10;Improvement over Random: 28.3%&#10;Selection time per 1k features: 38.3 s&#10;Datasets: 71&#10;Next to MI: 28.0%, 46.6 s'/><circle class='sa-hit' cx='526.6' cy='44.3' r='10.5' data-tip='MI&#10;Improvement over Random: 28.0%&#10;Selection time per 1k features: 46.6 s&#10;Datasets: 71&#10;Next to RFImportance: 28.3%, 38.3 s' aria-label='MI&#10;Improvement over Random: 28.0%&#10;Selection time per 1k features: 46.6 s&#10;Datasets: 71&#10;Next to RFImportance: 28.3%, 38.3 s'/><circle class='sa-hit' cx='476.4' cy='55.5' r='10.5' data-tip='CART&#10;Improvement over Random: 27.0%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to mRMR: 26.3%, 16.2 s&#10;Next to MarkovBlanket: 25.4%, 19 s' aria-label='CART&#10;Improvement over Random: 27.0%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to mRMR: 26.3%, 16.2 s&#10;Next to MarkovBlanket: 25.4%, 19 s'/><circle class='sa-hit' cx='481.0' cy='62.6' r='10.5' data-tip='mRMR&#10;Improvement over Random: 26.3%&#10;Selection time per 1k features: 16.2 s&#10;Datasets: 71&#10;Next to CART: 27.0%, 14.5 s&#10;Next to MarkovBlanket: 25.4%, 19 s' aria-label='mRMR&#10;Improvement over Random: 26.3%&#10;Selection time per 1k features: 16.2 s&#10;Datasets: 71&#10;Next to CART: 27.0%, 14.5 s&#10;Next to MarkovBlanket: 25.4%, 19 s'/><circle class='sa-hit' cx='691.8' cy='64.5' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 26.2%&#10;Selection time per 1k features: 2.14e+03 s&#10;Datasets: 71&#10;Next to Lasso: 24.5%, 1.97e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 26.2%&#10;Selection time per 1k features: 2.14e+03 s&#10;Datasets: 71&#10;Next to Lasso: 24.5%, 1.97e+03 s'/><circle class='sa-hit' cx='487.9' cy='72.6' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 25.4%&#10;Selection time per 1k features: 19 s&#10;Datasets: 71&#10;Next to CART: 27.0%, 14.5 s&#10;Next to mRMR: 26.3%, 16.2 s' aria-label='MarkovBlanket&#10;Improvement over Random: 25.4%&#10;Selection time per 1k features: 19 s&#10;Datasets: 71&#10;Next to CART: 27.0%, 14.5 s&#10;Next to mRMR: 26.3%, 16.2 s'/><circle class='sa-hit' cx='688.2' cy='82.6' r='10.5' data-tip='Lasso&#10;Improvement over Random: 24.5%&#10;Selection time per 1k features: 1.97e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 26.2%, 2.14e+03 s' aria-label='Lasso&#10;Improvement over Random: 24.5%&#10;Selection time per 1k features: 1.97e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 26.2%, 2.14e+03 s'/><circle class='sa-hit' cx='782.3' cy='96.1' r='10.5' data-tip='SFS&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 1.75e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 1.75e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='471.1' cy='104.4' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 22.5%&#10;Selection time per 1k features: 12.8 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 22.5%&#10;Selection time per 1k features: 12.8 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.8' cy='154.0' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='738.2' cy='184.3' r='10.5' data-tip='LOCO&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 6.29e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 6.29e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='255.6' r='10.5' data-tip='SBE&#10;Improvement over Random: 8.8%&#10;Selection time per 1k features: 3.5e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 8.8%&#10;Selection time per 1k features: 3.5e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.2' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0025 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0025 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  F-Test,28.7686,0.6293,71,True
37
  RFImportance,28.295,38.3008,71,False
38
  MI,27.9958,46.5786,71,False
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='162.3' x2='162.3' y1='20.0' y2='368.0'/><text class='sa-text' x='162.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='261.6' x2='261.6' y1='20.0' y2='368.0'/><text class='sa-text' x='261.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='360.9' x2='360.9' y1='20.0' y2='368.0'/><text class='sa-text' x='360.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='460.3' x2='460.3' y1='20.0' y2='368.0'/><text class='sa-text' x='460.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.6' x2='559.6' y1='20.0' y2='368.0'/><text class='sa-text' x='559.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='658.9' x2='658.9' y1='20.0' y2='368.0'/><text class='sa-text' x='658.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='758.3' x2='758.3' y1='20.0' y2='368.0'/><text class='sa-text' x='758.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.2' y2='352.2'/><text class='sa-text' x='58.0' y='358.2' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='297.2' y2='297.2'/><text class='sa-text' x='58.0' y='303.2' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='242.2' y2='242.2'/><text class='sa-text' x='58.0' y='248.2' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='187.2' y2='187.2'/><text class='sa-text' x='58.0' y='193.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='132.2' y2='132.2'/><text class='sa-text' x='58.0' y='138.2' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='77.2' y2='77.2'/><text class='sa-text' x='58.0' y='83.2' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='22.2' y2='22.2'/><text class='sa-text' x='58.0' y='28.2' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='20.0' width='780.8' height='348.1'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 194.0) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='55.3' x2='69.0' y2='22.0' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='79.3' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.2 341.0,35.8' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='341.0' cy='35.8' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='518.2' cy='41.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='526.6' cy='44.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='476.4' cy='55.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='481.0' cy='62.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.8' cy='64.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='487.9' cy='72.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='688.2' cy='82.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='782.3' cy='96.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='471.1' cy='104.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.8' cy='154.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='738.2' cy='184.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='255.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.2' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='341.0' y1='35.8' x2='261.0' y2='31.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='261.0' y='31.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='518.2' y1='41.0' x2='422.5' y2='29.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='422.5' y='29.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>RFImportance</text><line class='sa-ink' x1='526.6' y1='44.3' x2='425.9' y2='29.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='425.9' y='29.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='476.4' y1='55.5' x2='428.5' y2='60.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='428.5' y='60.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='481.0' y1='62.6' x2='407.9' y2='93.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='407.9' y='93.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='691.8' y1='64.5' x2='738.4' y2='49.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='738.4' y='49.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='487.9' y1='72.6' x2='579.2' y2='94.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='579.2' y='94.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='688.2' y1='82.6' x2='740.1' y2='111.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='740.1' y='111.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='782.3' y1='96.1' x2='817.2' y2='79.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='817.2' y='79.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='471.1' y1='104.4' x2='526.5' y2='124.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='526.5' y='124.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='797.0' y='143.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='738.2' y1='184.3' x2='705.5' y2='197.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='705.5' y='197.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='812.3' y1='255.6' x2='787.1' y2='269.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='787.1' y='269.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.2' x2='156.2' y2='338.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='338.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='341.0' cy='35.8' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 28.8%&#10;Selection time per 1k features: 0.629 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 28.8%&#10;Selection time per 1k features: 0.629 s&#10;Datasets: 71'/><circle class='sa-hit' cx='518.2' cy='41.0' r='10.5' data-tip='RFImportance&#10;Improvement over Random: 28.3%&#10;Selection time per 1k features: 38.3 s&#10;Datasets: 71&#10;Next to MI: 28.0%, 46.6 s' aria-label='RFImportance&#10;Improvement over Random: 28.3%&#10;Selection time per 1k features: 38.3 s&#10;Datasets: 71&#10;Next to MI: 28.0%, 46.6 s'/><circle class='sa-hit' cx='526.6' cy='44.3' r='10.5' data-tip='MI&#10;Improvement over Random: 28.0%&#10;Selection time per 1k features: 46.6 s&#10;Datasets: 71&#10;Next to RFImportance: 28.3%, 38.3 s' aria-label='MI&#10;Improvement over Random: 28.0%&#10;Selection time per 1k features: 46.6 s&#10;Datasets: 71&#10;Next to RFImportance: 28.3%, 38.3 s'/><circle class='sa-hit' cx='476.4' cy='55.5' r='10.5' data-tip='CART&#10;Improvement over Random: 27.0%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to mRMR: 26.3%, 16.2 s&#10;Next to MarkovBlanket: 25.4%, 19 s' aria-label='CART&#10;Improvement over Random: 27.0%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to mRMR: 26.3%, 16.2 s&#10;Next to MarkovBlanket: 25.4%, 19 s'/><circle class='sa-hit' cx='481.0' cy='62.6' r='10.5' data-tip='mRMR&#10;Improvement over Random: 26.3%&#10;Selection time per 1k features: 16.2 s&#10;Datasets: 71&#10;Next to CART: 27.0%, 14.5 s&#10;Next to MarkovBlanket: 25.4%, 19 s' aria-label='mRMR&#10;Improvement over Random: 26.3%&#10;Selection time per 1k features: 16.2 s&#10;Datasets: 71&#10;Next to CART: 27.0%, 14.5 s&#10;Next to MarkovBlanket: 25.4%, 19 s'/><circle class='sa-hit' cx='691.8' cy='64.5' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 26.2%&#10;Selection time per 1k features: 2.14e+03 s&#10;Datasets: 71&#10;Next to Lasso: 24.5%, 1.97e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 26.2%&#10;Selection time per 1k features: 2.14e+03 s&#10;Datasets: 71&#10;Next to Lasso: 24.5%, 1.97e+03 s'/><circle class='sa-hit' cx='487.9' cy='72.6' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 25.4%&#10;Selection time per 1k features: 19 s&#10;Datasets: 71&#10;Next to CART: 27.0%, 14.5 s&#10;Next to mRMR: 26.3%, 16.2 s' aria-label='MarkovBlanket&#10;Improvement over Random: 25.4%&#10;Selection time per 1k features: 19 s&#10;Datasets: 71&#10;Next to CART: 27.0%, 14.5 s&#10;Next to mRMR: 26.3%, 16.2 s'/><circle class='sa-hit' cx='688.2' cy='82.6' r='10.5' data-tip='Lasso&#10;Improvement over Random: 24.5%&#10;Selection time per 1k features: 1.97e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 26.2%, 2.14e+03 s' aria-label='Lasso&#10;Improvement over Random: 24.5%&#10;Selection time per 1k features: 1.97e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 26.2%, 2.14e+03 s'/><circle class='sa-hit' cx='782.3' cy='96.1' r='10.5' data-tip='SFS&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 1.75e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 1.75e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='471.1' cy='104.4' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 22.5%&#10;Selection time per 1k features: 12.8 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 22.5%&#10;Selection time per 1k features: 12.8 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.8' cy='154.0' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 18.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='738.2' cy='184.3' r='10.5' data-tip='LOCO&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 6.29e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 6.29e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='255.6' r='10.5' data-tip='SBE&#10;Improvement over Random: 8.8%&#10;Selection time per 1k features: 3.5e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 8.8%&#10;Selection time per 1k features: 3.5e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.2' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0025 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0025 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  F-Test,28.7686,0.6293,71,True
37
  RFImportance,28.295,38.3008,71,False
38
  MI,27.9958,46.5786,71,False
data/subsets/tasks_all/models_linearmodel/budgets_1/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1405.0'>1405<span class='sub'>+59 / −56</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1377.0'>1377<span class='sub'>+57 / −56</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1365.0'>1365<span class='sub'>+61 / −51</span></td><td class='num' data-sort='27.1'>27.1</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1342.0'>1342<span class='sub'>+52 / −45</span></td><td class='num' data-sort='26.2'>26.2</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1326.0'>1326<span class='sub'>+43 / −46</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1318.0'>1318<span class='sub'>+53 / −49</span></td><td class='num' data-sort='24.1'>24.1</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1301.0'>1301<span class='sub'>+46 / −39</span></td><td class='num' data-sort='21.4'>21.4</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1296.0'>1296<span class='sub'>+45 / −44</span></td><td class='num' data-sort='24.4'>24.4</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1280.0'>1280<span class='sub'>+45 / −42</span></td><td class='num' data-sort='21.6'>21.6</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1267.0'>1267<span class='sub'>+51 / −49</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1791.6'>1,791.6</td><td class='num' data-sort='43.7'>43.7</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1216.0'>1216<span class='sub'>+48 / −46</span></td><td class='num' data-sort='16.8'>16.8</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='13.0'>13.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1199.0'>1199<span class='sub'>+46 / −46</span></td><td class='num' data-sort='12.5'>12.5</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='235.0'>235.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1184.0'>1184<span class='sub'>+50 / −53</span></td><td class='num' data-sort='10.8'>10.8</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3391.5'>3,391.5</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1026.0'>1026<span class='sub'>+56 / −73</span></td><td class='num' data-sort='-22.4'>-22.4</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+53 / −60</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,ElasticNet,Non-search,Regularization,1405.0,1349.0,1464.0,25.9,0.519,0.873,6.3,0.0
37
  2,Lasso,Non-search,Regularization,1377.0,1321.0,1434.0,25.5,0.422,0.825,2.0,0.0
38
  3,RFImportance,Non-search,Tree,1365.0,1314.0,1426.0,27.1,0.653,0.741,0.2,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1405.0'>1405<span class='sub'>+59 / −56</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1377.0'>1377<span class='sub'>+57 / −56</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1365.0'>1365<span class='sub'>+61 / −51</span></td><td class='num' data-sort='27.1'>27.1</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1342.0'>1342<span class='sub'>+52 / −45</span></td><td class='num' data-sort='26.2'>26.2</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1326.0'>1326<span class='sub'>+43 / −46</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1318.0'>1318<span class='sub'>+53 / −49</span></td><td class='num' data-sort='24.1'>24.1</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1301.0'>1301<span class='sub'>+46 / −39</span></td><td class='num' data-sort='21.4'>21.4</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1296.0'>1296<span class='sub'>+45 / −44</span></td><td class='num' data-sort='24.4'>24.4</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1280.0'>1280<span class='sub'>+45 / −42</span></td><td class='num' data-sort='21.6'>21.6</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1267.0'>1267<span class='sub'>+51 / −49</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1791.6'>1,791.6</td><td class='num' data-sort='43.7'>43.7</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1216.0'>1216<span class='sub'>+48 / −46</span></td><td class='num' data-sort='16.8'>16.8</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='13.0'>13.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1199.0'>1199<span class='sub'>+46 / −46</span></td><td class='num' data-sort='12.5'>12.5</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='235.0'>235.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1184.0'>1184<span class='sub'>+50 / −53</span></td><td class='num' data-sort='10.8'>10.8</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3391.5'>3,391.5</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1026.0'>1026<span class='sub'>+56 / −73</span></td><td class='num' data-sort='-22.4'>-22.4</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+53 / −60</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,ElasticNet,Non-search,Regularization,1405.0,1349.0,1464.0,25.9,0.519,0.873,6.3,0.0
39
  2,Lasso,Non-search,Regularization,1377.0,1321.0,1434.0,25.5,0.422,0.825,2.0,0.0
40
  3,RFImportance,Non-search,Tree,1365.0,1314.0,1426.0,27.1,0.653,0.741,0.2,0.0
data/subsets/tasks_all/models_linearmodel/budgets_1/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='159.3' x2='159.3' y1='16.2' y2='368.0'/><text class='sa-text' x='159.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='259.4' x2='259.4' y1='16.2' y2='368.0'/><text class='sa-text' x='259.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='359.5' x2='359.5' y1='16.2' y2='368.0'/><text class='sa-text' x='359.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='459.6' x2='459.6' y1='16.2' y2='368.0'/><text class='sa-text' x='459.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.7' x2='559.7' y1='16.2' y2='368.0'/><text class='sa-text' x='559.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='659.7' x2='659.7' y1='16.2' y2='368.0'/><text class='sa-text' x='659.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='759.8' x2='759.8' y1='16.2' y2='368.0'/><text class='sa-text' x='759.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='292.9' y2='292.9'/><text class='sa-text' x='57.9' y='298.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='233.8' y2='233.8'/><text class='sa-text' x='57.9' y='239.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='174.7' y2='174.7'/><text class='sa-text' x='57.9' y='180.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='115.6' y2='115.6'/><text class='sa-text' x='57.9' y='121.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='56.5' y2='56.5'/><text class='sa-text' x='57.9' y='62.5' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 340.6,50.3 518.4,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='518.4' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='526.0' cy='42.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.4' cy='46.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='340.6' cy='50.3' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='690.3' cy='50.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='492.1' cy='63.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='475.9' cy='66.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='469.8' cy='97.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='488.6' cy='99.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='799.7' cy='117.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.5' cy='154.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='740.2' cy='204.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='223.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='518.4' y1='32.2' x2='402.6' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='402.6' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='526.0' y1='42.3' x2='399.4' y2='59.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='399.4' y='59.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='691.4' y1='46.4' x2='738.2' y2='32.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='738.2' y='32.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='340.6' y1='50.3' x2='301.0' y2='64.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='301.0' y='64.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='690.3' y1='50.5' x2='656.8' y2='65.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='656.8' y='65.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='492.1' y1='63.3' x2='550.1' y2='56.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='550.1' y='56.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='475.9' y1='66.6' x2='426.8' y2='85.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='426.8' y='85.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='469.8' y1='97.3' x2='418.2' y2='121.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='418.2' y='121.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='488.6' y1='99.6' x2='601.3' y2='159.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='601.3' y='159.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='799.7' y1='117.6' x2='820.9' y2='103.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='820.9' y='103.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><text class='sa-dim sa-halo' x='796.9' y='135.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='740.2' y1='204.1' x2='706.2' y2='218.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='706.2' y='218.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='237.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='518.4' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 38.7 s&#10;Datasets: 71&#10;Next to MI: 26.2%, 46.1 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 38.7 s&#10;Datasets: 71&#10;Next to MI: 26.2%, 46.1 s'/><circle class='sa-hit' cx='526.0' cy='42.3' r='10.5' data-tip='MI&#10;Improvement over Random: 26.2%&#10;Selection time per 1k features: 46.1 s&#10;Datasets: 71&#10;Next to RFImportance: 27.1%, 38.7 s' aria-label='MI&#10;Improvement over Random: 26.2%&#10;Selection time per 1k features: 46.1 s&#10;Datasets: 71&#10;Next to RFImportance: 27.1%, 38.7 s'/><circle class='sa-hit' cx='691.4' cy='46.4' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 2.07e+03 s&#10;Datasets: 71&#10;Next to Lasso: 25.5%, 2.02e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 2.07e+03 s&#10;Datasets: 71&#10;Next to Lasso: 25.5%, 2.02e+03 s'/><circle class='sa-hit' cx='340.6' cy='50.3' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 0.648 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 0.648 s&#10;Datasets: 71'/><circle class='sa-hit' cx='690.3' cy='50.5' r='10.5' data-tip='Lasso&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 2.02e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.9%, 2.07e+03 s' aria-label='Lasso&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 2.02e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.9%, 2.07e+03 s'/><circle class='sa-hit' cx='492.1' cy='63.3' r='10.5' data-tip='mRMR&#10;Improvement over Random: 24.4%&#10;Selection time per 1k features: 21.2 s&#10;Datasets: 71&#10;Next to CART: 24.1%, 14.6 s' aria-label='mRMR&#10;Improvement over Random: 24.4%&#10;Selection time per 1k features: 21.2 s&#10;Datasets: 71&#10;Next to CART: 24.1%, 14.6 s'/><circle class='sa-hit' cx='475.9' cy='66.6' r='10.5' data-tip='CART&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 14.6 s&#10;Datasets: 71&#10;Next to mRMR: 24.4%, 21.2 s' aria-label='CART&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 14.6 s&#10;Datasets: 71&#10;Next to mRMR: 24.4%, 21.2 s'/><circle class='sa-hit' cx='469.8' cy='97.3' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 21.6%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 21.4%, 19.5 s' aria-label='GainRatio&#10;Improvement over Random: 21.6%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 21.4%, 19.5 s'/><circle class='sa-hit' cx='488.6' cy='99.6' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 21.4%&#10;Selection time per 1k features: 19.5 s&#10;Datasets: 71&#10;Next to GainRatio: 21.6%, 12.7 s' aria-label='MarkovBlanket&#10;Improvement over Random: 21.4%&#10;Selection time per 1k features: 19.5 s&#10;Datasets: 71&#10;Next to GainRatio: 21.6%, 12.7 s'/><circle class='sa-hit' cx='799.7' cy='117.6' r='10.5' data-tip='SFS&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 2.5e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 2.5e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='797.5' cy='154.0' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 16.8%&#10;Selection time per 1k features: 2.38e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 16.8%&#10;Selection time per 1k features: 2.38e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='740.2' cy='204.1' r='10.5' data-tip='LOCO&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 6.37e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 6.37e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='223.9' r='10.5' data-tip='SBE&#10;Improvement over Random: 10.8%&#10;Selection time per 1k features: 3.35e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 10.8%&#10;Selection time per 1k features: 3.35e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,27.0594,38.7475,71,True
37
  MI,26.2027,46.0868,71,False
38
  ElasticNet,25.8586,2071.727,71,False
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='159.3' x2='159.3' y1='16.2' y2='368.0'/><text class='sa-text' x='159.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='259.4' x2='259.4' y1='16.2' y2='368.0'/><text class='sa-text' x='259.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='359.5' x2='359.5' y1='16.2' y2='368.0'/><text class='sa-text' x='359.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='459.6' x2='459.6' y1='16.2' y2='368.0'/><text class='sa-text' x='459.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.7' x2='559.7' y1='16.2' y2='368.0'/><text class='sa-text' x='559.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='659.7' x2='659.7' y1='16.2' y2='368.0'/><text class='sa-text' x='659.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='759.8' x2='759.8' y1='16.2' y2='368.0'/><text class='sa-text' x='759.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='292.9' y2='292.9'/><text class='sa-text' x='57.9' y='298.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='233.8' y2='233.8'/><text class='sa-text' x='57.9' y='239.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='174.7' y2='174.7'/><text class='sa-text' x='57.9' y='180.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='115.6' y2='115.6'/><text class='sa-text' x='57.9' y='121.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='56.5' y2='56.5'/><text class='sa-text' x='57.9' y='62.5' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 340.6,50.3 518.4,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='518.4' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='526.0' cy='42.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.4' cy='46.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='340.6' cy='50.3' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='690.3' cy='50.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='492.1' cy='63.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='475.9' cy='66.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='469.8' cy='97.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='488.6' cy='99.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='799.7' cy='117.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.5' cy='154.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='740.2' cy='204.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='223.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='518.4' y1='32.2' x2='403.2' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='403.2' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='526.0' y1='42.3' x2='412.6' y2='53.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='412.6' y='53.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='691.4' y1='46.4' x2='738.2' y2='32.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='738.2' y='32.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='340.6' y1='50.3' x2='302.2' y2='64.0' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='302.2' y='64.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='690.3' y1='50.5' x2='658.0' y2='64.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='658.0' y='64.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='492.1' y1='63.3' x2='551.3' y2='57.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='551.3' y='57.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='475.9' y1='66.6' x2='431.8' y2='83.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='431.8' y='83.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='469.8' y1='97.3' x2='433.3' y2='122.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='433.3' y='122.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='488.6' y1='99.6' x2='590.0' y2='146.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='590.0' y='146.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='799.7' y1='117.6' x2='820.9' y2='103.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='820.9' y='103.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><text class='sa-dim sa-halo' x='796.9' y='139.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='740.2' y1='204.1' x2='707.5' y2='217.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='707.5' y='217.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='238.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='518.4' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 38.7 s&#10;Datasets: 71&#10;Next to MI: 26.2%, 46.1 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 38.7 s&#10;Datasets: 71&#10;Next to MI: 26.2%, 46.1 s'/><circle class='sa-hit' cx='526.0' cy='42.3' r='10.5' data-tip='MI&#10;Improvement over Random: 26.2%&#10;Selection time per 1k features: 46.1 s&#10;Datasets: 71&#10;Next to RFImportance: 27.1%, 38.7 s' aria-label='MI&#10;Improvement over Random: 26.2%&#10;Selection time per 1k features: 46.1 s&#10;Datasets: 71&#10;Next to RFImportance: 27.1%, 38.7 s'/><circle class='sa-hit' cx='691.4' cy='46.4' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 2.07e+03 s&#10;Datasets: 71&#10;Next to Lasso: 25.5%, 2.02e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 2.07e+03 s&#10;Datasets: 71&#10;Next to Lasso: 25.5%, 2.02e+03 s'/><circle class='sa-hit' cx='340.6' cy='50.3' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 0.648 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 0.648 s&#10;Datasets: 71'/><circle class='sa-hit' cx='690.3' cy='50.5' r='10.5' data-tip='Lasso&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 2.02e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.9%, 2.07e+03 s' aria-label='Lasso&#10;Improvement over Random: 25.5%&#10;Selection time per 1k features: 2.02e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 25.9%, 2.07e+03 s'/><circle class='sa-hit' cx='492.1' cy='63.3' r='10.5' data-tip='mRMR&#10;Improvement over Random: 24.4%&#10;Selection time per 1k features: 21.2 s&#10;Datasets: 71&#10;Next to CART: 24.1%, 14.6 s' aria-label='mRMR&#10;Improvement over Random: 24.4%&#10;Selection time per 1k features: 21.2 s&#10;Datasets: 71&#10;Next to CART: 24.1%, 14.6 s'/><circle class='sa-hit' cx='475.9' cy='66.6' r='10.5' data-tip='CART&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 14.6 s&#10;Datasets: 71&#10;Next to mRMR: 24.4%, 21.2 s' aria-label='CART&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 14.6 s&#10;Datasets: 71&#10;Next to mRMR: 24.4%, 21.2 s'/><circle class='sa-hit' cx='469.8' cy='97.3' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 21.6%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 21.4%, 19.5 s' aria-label='GainRatio&#10;Improvement over Random: 21.6%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 21.4%, 19.5 s'/><circle class='sa-hit' cx='488.6' cy='99.6' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 21.4%&#10;Selection time per 1k features: 19.5 s&#10;Datasets: 71&#10;Next to GainRatio: 21.6%, 12.7 s' aria-label='MarkovBlanket&#10;Improvement over Random: 21.4%&#10;Selection time per 1k features: 19.5 s&#10;Datasets: 71&#10;Next to GainRatio: 21.6%, 12.7 s'/><circle class='sa-hit' cx='799.7' cy='117.6' r='10.5' data-tip='SFS&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 2.5e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 19.8%&#10;Selection time per 1k features: 2.5e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='797.5' cy='154.0' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 16.8%&#10;Selection time per 1k features: 2.38e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 16.8%&#10;Selection time per 1k features: 2.38e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='740.2' cy='204.1' r='10.5' data-tip='LOCO&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 6.37e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 6.37e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='223.9' r='10.5' data-tip='SBE&#10;Improvement over Random: 10.8%&#10;Selection time per 1k features: 3.35e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 10.8%&#10;Selection time per 1k features: 3.35e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,27.0594,38.7475,71,True
37
  MI,26.2027,46.0868,71,False
38
  ElasticNet,25.8586,2071.727,71,False
data/subsets/tasks_all/models_linearmodel/budgets_2/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1313.0'>1313<span class='sub'>+63 / −56</span></td><td class='num' data-sort='20.6'>20.6</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+57 / −55</span></td><td class='num' data-sort='19.4'>19.4</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1263.0'>1263<span class='sub'>+58 / −50</span></td><td class='num' data-sort='22.2'>22.2</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1258.0'>1258<span class='sub'>+45 / −45</span></td><td class='num' data-sort='20.9'>20.9</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1216.0'>1216<span class='sub'>+46 / −45</span></td><td class='num' data-sort='18.6'>18.6</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1203.0'>1203<span class='sub'>+55 / −49</span></td><td class='num' data-sort='17.5'>17.5</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1185.0'>1185<span class='sub'>+45 / −46</span></td><td class='num' data-sort='15.0'>15.0</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1173.0'>1173<span class='sub'>+48 / −50</span></td><td class='num' data-sort='17.9'>17.9</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+42 / −43</span></td><td class='num' data-sort='14.8'>14.8</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+43 / −46</span></td><td class='num' data-sort='13.8'>13.8</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2264.7'>2,264.7</td><td class='num' data-sort='47.9'>47.9</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+44 / −45</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='12.8'>12.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1109.0'>1109<span class='sub'>+41 / −48</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='6.1'>6.1</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1088.0'>1088<span class='sub'>+47 / −49</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2787.0'>2,787.0</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+56 / −65</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='942.0'>942<span class='sub'>+55 / −74</span></td><td class='num' data-sort='-52.8'>-52.8</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,ElasticNet,Non-search,Regularization,1313.0,1257.0,1376.0,20.6,0.481,0.777,6.3,0.0
37
  2,Lasso,Non-search,Regularization,1274.0,1219.0,1331.0,19.4,0.389,0.73,1.8,0.0
38
  3,RFImportance,Non-search,Tree,1263.0,1213.0,1321.0,22.2,0.676,0.691,0.2,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1313.0'>1313<span class='sub'>+63 / −56</span></td><td class='num' data-sort='20.6'>20.6</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+57 / −55</span></td><td class='num' data-sort='19.4'>19.4</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1263.0'>1263<span class='sub'>+58 / −50</span></td><td class='num' data-sort='22.2'>22.2</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1258.0'>1258<span class='sub'>+45 / −45</span></td><td class='num' data-sort='20.9'>20.9</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1216.0'>1216<span class='sub'>+46 / −45</span></td><td class='num' data-sort='18.6'>18.6</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1203.0'>1203<span class='sub'>+55 / −49</span></td><td class='num' data-sort='17.5'>17.5</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1185.0'>1185<span class='sub'>+45 / −46</span></td><td class='num' data-sort='15.0'>15.0</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1173.0'>1173<span class='sub'>+48 / −50</span></td><td class='num' data-sort='17.9'>17.9</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+42 / −43</span></td><td class='num' data-sort='14.8'>14.8</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+43 / −46</span></td><td class='num' data-sort='13.8'>13.8</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2264.7'>2,264.7</td><td class='num' data-sort='47.9'>47.9</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+44 / −45</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='12.8'>12.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1109.0'>1109<span class='sub'>+41 / −48</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='6.1'>6.1</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1088.0'>1088<span class='sub'>+47 / −49</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2787.0'>2,787.0</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+56 / −65</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='942.0'>942<span class='sub'>+55 / −74</span></td><td class='num' data-sort='-52.8'>-52.8</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,ElasticNet,Non-search,Regularization,1313.0,1257.0,1376.0,20.6,0.481,0.777,6.3,0.0
39
  2,Lasso,Non-search,Regularization,1274.0,1219.0,1331.0,19.4,0.389,0.73,1.8,0.0
40
  3,RFImportance,Non-search,Tree,1263.0,1213.0,1321.0,22.2,0.676,0.691,0.2,0.0
data/subsets/tasks_all/models_linearmodel/budgets_2/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='161.2' x2='161.2' y1='16.2' y2='368.0'/><text class='sa-text' x='161.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='261.7' x2='261.7' y1='16.2' y2='368.0'/><text class='sa-text' x='261.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.2' x2='362.2' y1='16.2' y2='368.0'/><text class='sa-text' x='362.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='462.7' x2='462.7' y1='16.2' y2='368.0'/><text class='sa-text' x='462.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='563.2' x2='563.2' y1='16.2' y2='368.0'/><text class='sa-text' x='563.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.7' x2='663.7' y1='16.2' y2='368.0'/><text class='sa-text' x='663.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='764.2' x2='764.2' y1='16.2' y2='368.0'/><text class='sa-text' x='764.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='280.1' y2='280.1'/><text class='sa-text' x='57.9' y='286.1' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='208.1' y2='208.1'/><text class='sa-text' x='57.9' y='214.1' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='136.2' y2='136.2'/><text class='sa-text' x='57.9' y='142.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='64.2' y2='64.2'/><text class='sa-text' x='57.9' y='70.2' text-anchor='end' font-size='15'>20</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 342.7,84.9 521.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='521.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='529.2' cy='51.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='697.9' cy='55.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='73.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='342.7' cy='84.9' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='514.8' cy='93.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='802.2' cy='97.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='478.7' cy='99.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='473.1' cy='136.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='490.5' cy='139.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='154.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='744.6' cy='235.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='811.9' cy='243.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='521.5' y1='32.2' x2='399.2' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='399.2' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='529.2' y1='51.4' x2='481.9' y2='57.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='481.9' y='57.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='697.9' y1='55.2' x2='755.7' y2='40.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='755.7' y='40.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='695.4' y1='73.4' x2='649.8' y2='82.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='649.8' y='82.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='342.7' y1='84.9' x2='303.0' y2='99.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='303.0' y='99.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='514.8' y1='93.9' x2='556.4' y2='79.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='556.4' y='79.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='802.2' y1='97.1' x2='779.9' y2='116.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='779.9' y='116.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='478.7' y1='99.5' x2='437.8' y2='111.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='437.8' y='111.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='473.1' y1='136.8' x2='397.6' y2='148.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='397.6' y='148.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='490.5' y1='139.5' x2='581.1' y2='177.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='581.1' y='177.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='812.3' y1='154.2' x2='828.6' y2='167.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='167.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='744.6' y1='235.7' x2='767.9' y2='221.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='767.9' y='221.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='257.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='521.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 22.2%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71&#10;Next to MI: 20.9%, 45.9 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 22.2%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71&#10;Next to MI: 20.9%, 45.9 s'/><circle class='sa-hit' cx='529.2' cy='51.4' r='10.5' data-tip='MI&#10;Improvement over Random: 20.9%&#10;Selection time per 1k features: 45.9 s&#10;Datasets: 71&#10;Next to RFImportance: 22.2%, 38.5 s' aria-label='MI&#10;Improvement over Random: 20.9%&#10;Selection time per 1k features: 45.9 s&#10;Datasets: 71&#10;Next to RFImportance: 22.2%, 38.5 s'/><circle class='sa-hit' cx='697.9' cy='55.2' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 20.6%&#10;Selection time per 1k features: 2.19e+03 s&#10;Datasets: 71&#10;Next to Lasso: 19.4%, 2.07e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 20.6%&#10;Selection time per 1k features: 2.19e+03 s&#10;Datasets: 71&#10;Next to Lasso: 19.4%, 2.07e+03 s'/><circle class='sa-hit' cx='695.4' cy='73.4' r='10.5' data-tip='Lasso&#10;Improvement over Random: 19.4%&#10;Selection time per 1k features: 2.07e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 20.6%, 2.19e+03 s' aria-label='Lasso&#10;Improvement over Random: 19.4%&#10;Selection time per 1k features: 2.07e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 20.6%, 2.19e+03 s'/><circle class='sa-hit' cx='342.7' cy='84.9' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 18.6%&#10;Selection time per 1k features: 0.639 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 18.6%&#10;Selection time per 1k features: 0.639 s&#10;Datasets: 71'/><circle class='sa-hit' cx='514.8' cy='93.9' r='10.5' data-tip='mRMR&#10;Improvement over Random: 17.9%&#10;Selection time per 1k features: 33 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 17.9%&#10;Selection time per 1k features: 33 s&#10;Datasets: 71'/><circle class='sa-hit' cx='802.2' cy='97.1' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='478.7' cy='99.5' r='10.5' data-tip='CART&#10;Improvement over Random: 17.5%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71' aria-label='CART&#10;Improvement over Random: 17.5%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71'/><circle class='sa-hit' cx='473.1' cy='136.8' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 15.0%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 14.8%, 18.9 s' aria-label='GainRatio&#10;Improvement over Random: 15.0%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 14.8%, 18.9 s'/><circle class='sa-hit' cx='490.5' cy='139.5' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 14.8%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 15.0%, 12.7 s' aria-label='MarkovBlanket&#10;Improvement over Random: 14.8%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 15.0%, 12.7 s'/><circle class='sa-hit' cx='812.3' cy='154.2' r='10.5' data-tip='SFS&#10;Improvement over Random: 13.8%&#10;Selection time per 1k features: 3.01e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 13.8%&#10;Selection time per 1k features: 3.01e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='744.6' cy='235.7' r='10.5' data-tip='LOCO&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 6.39e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 6.39e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='811.9' cy='243.9' r='10.5' data-tip='SBE&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 2.99e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 2.99e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,22.2269,38.5063,71,True
37
  MI,20.8936,45.8704,71,False
38
  ElasticNet,20.6265,2189.5545,71,False
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='161.2' x2='161.2' y1='16.2' y2='368.0'/><text class='sa-text' x='161.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='261.7' x2='261.7' y1='16.2' y2='368.0'/><text class='sa-text' x='261.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.2' x2='362.2' y1='16.2' y2='368.0'/><text class='sa-text' x='362.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='462.7' x2='462.7' y1='16.2' y2='368.0'/><text class='sa-text' x='462.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='563.2' x2='563.2' y1='16.2' y2='368.0'/><text class='sa-text' x='563.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.7' x2='663.7' y1='16.2' y2='368.0'/><text class='sa-text' x='663.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='764.2' x2='764.2' y1='16.2' y2='368.0'/><text class='sa-text' x='764.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='280.1' y2='280.1'/><text class='sa-text' x='57.9' y='286.1' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='208.1' y2='208.1'/><text class='sa-text' x='57.9' y='214.1' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='136.2' y2='136.2'/><text class='sa-text' x='57.9' y='142.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='64.2' y2='64.2'/><text class='sa-text' x='57.9' y='70.2' text-anchor='end' font-size='15'>20</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 342.7,84.9 521.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='521.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='529.2' cy='51.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='697.9' cy='55.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='73.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='342.7' cy='84.9' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='514.8' cy='93.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='802.2' cy='97.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='478.7' cy='99.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='473.1' cy='136.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='490.5' cy='139.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='154.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='744.6' cy='235.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='811.9' cy='243.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='521.5' y1='32.2' x2='398.5' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='398.5' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='529.2' y1='51.4' x2='484.5' y2='53.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='484.5' y='53.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='697.9' y1='55.2' x2='755.7' y2='40.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='755.7' y='40.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='695.4' y1='73.4' x2='652.3' y2='82.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='652.3' y='82.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='342.7' y1='84.9' x2='303.0' y2='99.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='303.0' y='99.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='514.8' y1='93.9' x2='554.5' y2='79.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='554.5' y='79.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='802.2' y1='97.1' x2='785.0' y2='116.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='785.0' y='116.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='478.7' y1='99.5' x2='438.4' y2='113.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='438.4' y='113.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='473.1' y1='136.8' x2='392.5' y2='143.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='392.5' y='143.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='490.5' y1='139.5' x2='585.5' y2='181.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='585.5' y='181.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='812.3' y1='154.2' x2='828.6' y2='167.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='167.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='744.6' y1='235.7' x2='767.9' y2='221.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='767.9' y='221.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='257.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='521.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 22.2%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71&#10;Next to MI: 20.9%, 45.9 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 22.2%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71&#10;Next to MI: 20.9%, 45.9 s'/><circle class='sa-hit' cx='529.2' cy='51.4' r='10.5' data-tip='MI&#10;Improvement over Random: 20.9%&#10;Selection time per 1k features: 45.9 s&#10;Datasets: 71&#10;Next to RFImportance: 22.2%, 38.5 s' aria-label='MI&#10;Improvement over Random: 20.9%&#10;Selection time per 1k features: 45.9 s&#10;Datasets: 71&#10;Next to RFImportance: 22.2%, 38.5 s'/><circle class='sa-hit' cx='697.9' cy='55.2' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 20.6%&#10;Selection time per 1k features: 2.19e+03 s&#10;Datasets: 71&#10;Next to Lasso: 19.4%, 2.07e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 20.6%&#10;Selection time per 1k features: 2.19e+03 s&#10;Datasets: 71&#10;Next to Lasso: 19.4%, 2.07e+03 s'/><circle class='sa-hit' cx='695.4' cy='73.4' r='10.5' data-tip='Lasso&#10;Improvement over Random: 19.4%&#10;Selection time per 1k features: 2.07e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 20.6%, 2.19e+03 s' aria-label='Lasso&#10;Improvement over Random: 19.4%&#10;Selection time per 1k features: 2.07e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 20.6%, 2.19e+03 s'/><circle class='sa-hit' cx='342.7' cy='84.9' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 18.6%&#10;Selection time per 1k features: 0.639 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 18.6%&#10;Selection time per 1k features: 0.639 s&#10;Datasets: 71'/><circle class='sa-hit' cx='514.8' cy='93.9' r='10.5' data-tip='mRMR&#10;Improvement over Random: 17.9%&#10;Selection time per 1k features: 33 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 17.9%&#10;Selection time per 1k features: 33 s&#10;Datasets: 71'/><circle class='sa-hit' cx='802.2' cy='97.1' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='478.7' cy='99.5' r='10.5' data-tip='CART&#10;Improvement over Random: 17.5%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71' aria-label='CART&#10;Improvement over Random: 17.5%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71'/><circle class='sa-hit' cx='473.1' cy='136.8' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 15.0%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 14.8%, 18.9 s' aria-label='GainRatio&#10;Improvement over Random: 15.0%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 14.8%, 18.9 s'/><circle class='sa-hit' cx='490.5' cy='139.5' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 14.8%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 15.0%, 12.7 s' aria-label='MarkovBlanket&#10;Improvement over Random: 14.8%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 15.0%, 12.7 s'/><circle class='sa-hit' cx='812.3' cy='154.2' r='10.5' data-tip='SFS&#10;Improvement over Random: 13.8%&#10;Selection time per 1k features: 3.01e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 13.8%&#10;Selection time per 1k features: 3.01e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='744.6' cy='235.7' r='10.5' data-tip='LOCO&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 6.39e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 8.1%&#10;Selection time per 1k features: 6.39e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='811.9' cy='243.9' r='10.5' data-tip='SBE&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 2.99e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 2.99e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,22.2269,38.5063,71,True
37
  MI,20.8936,45.8704,71,False
38
  ElasticNet,20.6265,2189.5545,71,False
data/subsets/tasks_all/models_linearmodel/budgets_3/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1246.0'>1246<span class='sub'>+54 / −51</span></td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1237.0'>1237<span class='sub'>+55 / −48</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1223.0'>1223<span class='sub'>+63 / −54</span></td><td class='num' data-sort='12.2'>12.2</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1213.0'>1213<span class='sub'>+49 / −47</span></td><td class='num' data-sort='12.5'>12.5</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+45 / −46</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1162.0'>1162<span class='sub'>+49 / −44</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1152.0'>1152<span class='sub'>+47 / −47</span></td><td class='num' data-sort='11.8'>11.8</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1130.0'>1130<span class='sub'>+45 / −50</span></td><td class='num' data-sort='7.8'>7.8</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2680.3'>2,680.3</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1124.0'>1124<span class='sub'>+45 / −40</span></td><td class='num' data-sort='9.2'>9.2</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1121.0'>1121<span class='sub'>+49 / −57</span></td><td class='num' data-sort='5.8'>5.8</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1866.7'>1,866.7</td><td class='num' data-sort='45.1'>45.1</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1119.0'>1119<span class='sub'>+46 / −43</span></td><td class='num' data-sort='9.2'>9.2</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1108.0'>1108<span class='sub'>+42 / −46</span></td><td class='num' data-sort='6.5'>6.5</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='246.0'>246.0</td><td class='num' data-sort='11.7'>11.7</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1106.0'>1106<span class='sub'>+46 / −48</span></td><td class='num' data-sort='7.8'>7.8</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+47 / −53</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='977.0'>977<span class='sub'>+58 / −69</span></td><td class='num' data-sort='-21.5'>-21.5</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1246.0,1195.0,1300.0,15.9,0.661,0.6,0.2,0.0
37
  2,ElasticNet,Non-search,Regularization,1237.0,1189.0,1292.0,13.7,0.419,0.665,6.2,0.0
38
  3,Lasso,Non-search,Regularization,1223.0,1169.0,1286.0,12.2,0.334,0.643,1.9,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1246.0'>1246<span class='sub'>+54 / −51</span></td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1237.0'>1237<span class='sub'>+55 / −48</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1223.0'>1223<span class='sub'>+63 / −54</span></td><td class='num' data-sort='12.2'>12.2</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1213.0'>1213<span class='sub'>+49 / −47</span></td><td class='num' data-sort='12.5'>12.5</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+45 / −46</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1162.0'>1162<span class='sub'>+49 / −44</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1152.0'>1152<span class='sub'>+47 / −47</span></td><td class='num' data-sort='11.8'>11.8</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1130.0'>1130<span class='sub'>+45 / −50</span></td><td class='num' data-sort='7.8'>7.8</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2680.3'>2,680.3</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1124.0'>1124<span class='sub'>+45 / −40</span></td><td class='num' data-sort='9.2'>9.2</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1121.0'>1121<span class='sub'>+49 / −57</span></td><td class='num' data-sort='5.8'>5.8</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1866.7'>1,866.7</td><td class='num' data-sort='45.1'>45.1</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1119.0'>1119<span class='sub'>+46 / −43</span></td><td class='num' data-sort='9.2'>9.2</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1108.0'>1108<span class='sub'>+42 / −46</span></td><td class='num' data-sort='6.5'>6.5</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='246.0'>246.0</td><td class='num' data-sort='11.7'>11.7</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1106.0'>1106<span class='sub'>+46 / −48</span></td><td class='num' data-sort='7.8'>7.8</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+47 / −53</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='977.0'>977<span class='sub'>+58 / −69</span></td><td class='num' data-sort='-21.5'>-21.5</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1246.0,1195.0,1300.0,15.9,0.661,0.6,0.2,0.0
39
  2,ElasticNet,Non-search,Regularization,1237.0,1189.0,1292.0,13.7,0.419,0.665,6.2,0.0
40
  3,Lasso,Non-search,Regularization,1223.0,1169.0,1286.0,12.2,0.334,0.643,1.9,0.0
data/subsets/tasks_all/models_linearmodel/budgets_3/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='160.8' x2='160.8' y1='16.2' y2='368.0'/><text class='sa-text' x='160.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='260.5' x2='260.5' y1='16.2' y2='368.0'/><text class='sa-text' x='260.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='360.3' x2='360.3' y1='16.2' y2='368.0'/><text class='sa-text' x='360.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='460.1' x2='460.1' y1='16.2' y2='368.0'/><text class='sa-text' x='460.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.8' x2='559.8' y1='16.2' y2='368.0'/><text class='sa-text' x='559.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='659.6' x2='659.6' y1='16.2' y2='368.0'/><text class='sa-text' x='659.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='759.4' x2='759.4' y1='16.2' y2='368.0'/><text class='sa-text' x='759.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='311.7' y2='311.7'/><text class='sa-text' x='57.9' y='317.7' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='271.4' y2='271.4'/><text class='sa-text' x='57.9' y='277.4' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='231.0' y2='231.0'/><text class='sa-text' x='57.9' y='237.0' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='190.7' y2='190.7'/><text class='sa-text' x='57.9' y='196.7' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='150.4' y2='150.4'/><text class='sa-text' x='57.9' y='156.4' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='110.0' y2='110.0'/><text class='sa-text' x='57.9' y='116.0' text-anchor='end' font-size='15'>12</text><line class='sa-grid' x1='66.9' x2='847.8' y1='69.7' y2='69.7'/><text class='sa-text' x='57.9' y='75.7' text-anchor='end' font-size='15'>14</text><line class='sa-grid' x1='66.9' x2='847.8' y1='29.3' y2='29.3'/><text class='sa-text' x='57.9' y='35.3' text-anchor='end' font-size='15'>16</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 343.1,127.5 520.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='520.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='691.8' cy='76.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='526.6' cy='100.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='690.9' cy='106.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='540.1' cy='114.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='343.1' cy='127.5' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='476.8' cy='128.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='470.6' cy='165.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.8' cy='167.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='195.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='486.2' cy='195.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='740.3' cy='220.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.4' cy='234.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='520.8' y1='32.2' x2='396.1' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='396.1' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='691.8' y1='76.7' x2='743.6' y2='62.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='743.6' y='62.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='541.1' y='86.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='690.9' y1='106.5' x2='663.7' y2='119.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='663.7' y='119.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='540.1' y1='114.1' x2='556.5' y2='130.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='556.5' y='130.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='343.1' y1='127.5' x2='302.1' y2='141.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='302.1' y='141.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='476.8' y1='128.8' x2='440.9' y2='143.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='440.9' y='143.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='470.6' y1='165.9' x2='406.4' y2='179.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='406.4' y='179.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='796.9' y='153.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='195.4' x2='828.6' y2='209.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='209.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='486.2' y1='195.7' x2='592.0' y2='209.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='592.0' y='209.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><text class='sa-dim sa-halo' x='754.8' y='205.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='797.4' y1='234.4' x2='823.9' y2='248.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='823.9' y='248.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='175.1' y2='358.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='175.1' y='358.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='520.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 15.9%&#10;Selection time per 1k features: 40.6 s&#10;Datasets: 70' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 15.9%&#10;Selection time per 1k features: 40.6 s&#10;Datasets: 70'/><circle class='sa-hit' cx='691.8' cy='76.7' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 70' aria-label='ElasticNet&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 70'/><circle class='sa-hit' cx='526.6' cy='100.9' r='10.5' data-tip='MI&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 46.4 s&#10;Datasets: 70&#10;Next to mRMR: 11.8%, 63.4 s' aria-label='MI&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 46.4 s&#10;Datasets: 70&#10;Next to mRMR: 11.8%, 63.4 s'/><circle class='sa-hit' cx='690.9' cy='106.5' r='10.5' data-tip='Lasso&#10;Improvement over Random: 12.2%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 70' aria-label='Lasso&#10;Improvement over Random: 12.2%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 70'/><circle class='sa-hit' cx='540.1' cy='114.1' r='10.5' data-tip='mRMR&#10;Improvement over Random: 11.8%&#10;Selection time per 1k features: 63.4 s&#10;Datasets: 70&#10;Next to MI: 12.5%, 46.4 s' aria-label='mRMR&#10;Improvement over Random: 11.8%&#10;Selection time per 1k features: 63.4 s&#10;Datasets: 70&#10;Next to MI: 12.5%, 46.4 s'/><circle class='sa-hit' cx='343.1' cy='127.5' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 0.671 s&#10;Datasets: 70' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 0.671 s&#10;Datasets: 70'/><circle class='sa-hit' cx='476.8' cy='128.8' r='10.5' data-tip='CART&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 14.7 s&#10;Datasets: 70' aria-label='CART&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 14.7 s&#10;Datasets: 70'/><circle class='sa-hit' cx='470.6' cy='165.9' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 12.8 s&#10;Datasets: 70' aria-label='GainRatio&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 12.8 s&#10;Datasets: 70'/><circle class='sa-hit' cx='797.8' cy='167.4' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 2.42e+04 s&#10;Datasets: 70' aria-label='(R)ReliefF&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 2.42e+04 s&#10;Datasets: 70'/><circle class='sa-hit' cx='812.3' cy='195.4' r='10.5' data-tip='SFS&#10;Improvement over Random: 7.8%&#10;Selection time per 1k features: 3.39e+04 s&#10;Datasets: 70' aria-label='SFS&#10;Improvement over Random: 7.8%&#10;Selection time per 1k features: 3.39e+04 s&#10;Datasets: 70'/><circle class='sa-hit' cx='486.2' cy='195.7' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 7.8%&#10;Selection time per 1k features: 18.3 s&#10;Datasets: 70' aria-label='MarkovBlanket&#10;Improvement over Random: 7.8%&#10;Selection time per 1k features: 18.3 s&#10;Datasets: 70'/><circle class='sa-hit' cx='740.3' cy='220.0' r='10.5' data-tip='LOCO&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 6.43e+03 s&#10;Datasets: 70' aria-label='LOCO&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 6.43e+03 s&#10;Datasets: 70'/><circle class='sa-hit' cx='797.4' cy='234.4' r='10.5' data-tip='SBE&#10;Improvement over Random: 5.8%&#10;Selection time per 1k features: 2.41e+04 s&#10;Datasets: 70' aria-label='SBE&#10;Improvement over Random: 5.8%&#10;Selection time per 1k features: 2.41e+04 s&#10;Datasets: 70'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 70' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 70'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,15.8586,40.579,70,True
37
  ElasticNet,13.651,2100.4728,70,False
38
  MI,12.4516,46.4375,70,False
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='160.8' x2='160.8' y1='16.2' y2='368.0'/><text class='sa-text' x='160.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='260.5' x2='260.5' y1='16.2' y2='368.0'/><text class='sa-text' x='260.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='360.3' x2='360.3' y1='16.2' y2='368.0'/><text class='sa-text' x='360.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='460.1' x2='460.1' y1='16.2' y2='368.0'/><text class='sa-text' x='460.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.8' x2='559.8' y1='16.2' y2='368.0'/><text class='sa-text' x='559.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='659.6' x2='659.6' y1='16.2' y2='368.0'/><text class='sa-text' x='659.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='759.4' x2='759.4' y1='16.2' y2='368.0'/><text class='sa-text' x='759.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='311.7' y2='311.7'/><text class='sa-text' x='57.9' y='317.7' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='271.4' y2='271.4'/><text class='sa-text' x='57.9' y='277.4' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='231.0' y2='231.0'/><text class='sa-text' x='57.9' y='237.0' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='190.7' y2='190.7'/><text class='sa-text' x='57.9' y='196.7' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='150.4' y2='150.4'/><text class='sa-text' x='57.9' y='156.4' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='110.0' y2='110.0'/><text class='sa-text' x='57.9' y='116.0' text-anchor='end' font-size='15'>12</text><line class='sa-grid' x1='66.9' x2='847.8' y1='69.7' y2='69.7'/><text class='sa-text' x='57.9' y='75.7' text-anchor='end' font-size='15'>14</text><line class='sa-grid' x1='66.9' x2='847.8' y1='29.3' y2='29.3'/><text class='sa-text' x='57.9' y='35.3' text-anchor='end' font-size='15'>16</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 343.1,127.5 520.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='520.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='691.8' cy='76.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='526.6' cy='100.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='690.9' cy='106.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='540.1' cy='114.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='343.1' cy='127.5' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='476.8' cy='128.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='470.6' cy='165.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.8' cy='167.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='195.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='486.2' cy='195.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='740.3' cy='220.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.4' cy='234.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='520.8' y1='32.2' x2='398.6' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='398.6' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='691.8' y1='76.7' x2='743.6' y2='62.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='743.6' y='62.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='541.1' y='86.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='690.9' y1='106.5' x2='663.7' y2='119.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='663.7' y='119.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='540.1' y1='114.1' x2='556.5' y2='130.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='556.5' y='130.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='343.1' y1='127.5' x2='302.1' y2='141.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='302.1' y='141.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='476.8' y1='128.8' x2='440.9' y2='143.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='440.9' y='143.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='470.6' y1='165.9' x2='406.4' y2='179.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='406.4' y='179.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='796.9' y='153.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='195.4' x2='828.6' y2='209.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='209.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='486.2' y1='195.7' x2='588.2' y2='210.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='588.2' y='210.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><text class='sa-dim sa-halo' x='754.8' y='205.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='797.4' y1='234.4' x2='823.9' y2='248.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='823.9' y='248.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='175.1' y2='358.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='175.1' y='358.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='520.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 15.9%&#10;Selection time per 1k features: 40.6 s&#10;Datasets: 70' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 15.9%&#10;Selection time per 1k features: 40.6 s&#10;Datasets: 70'/><circle class='sa-hit' cx='691.8' cy='76.7' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 70' aria-label='ElasticNet&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 70'/><circle class='sa-hit' cx='526.6' cy='100.9' r='10.5' data-tip='MI&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 46.4 s&#10;Datasets: 70&#10;Next to mRMR: 11.8%, 63.4 s' aria-label='MI&#10;Improvement over Random: 12.5%&#10;Selection time per 1k features: 46.4 s&#10;Datasets: 70&#10;Next to mRMR: 11.8%, 63.4 s'/><circle class='sa-hit' cx='690.9' cy='106.5' r='10.5' data-tip='Lasso&#10;Improvement over Random: 12.2%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 70' aria-label='Lasso&#10;Improvement over Random: 12.2%&#10;Selection time per 1k features: 2.06e+03 s&#10;Datasets: 70'/><circle class='sa-hit' cx='540.1' cy='114.1' r='10.5' data-tip='mRMR&#10;Improvement over Random: 11.8%&#10;Selection time per 1k features: 63.4 s&#10;Datasets: 70&#10;Next to MI: 12.5%, 46.4 s' aria-label='mRMR&#10;Improvement over Random: 11.8%&#10;Selection time per 1k features: 63.4 s&#10;Datasets: 70&#10;Next to MI: 12.5%, 46.4 s'/><circle class='sa-hit' cx='343.1' cy='127.5' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 0.671 s&#10;Datasets: 70' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 0.671 s&#10;Datasets: 70'/><circle class='sa-hit' cx='476.8' cy='128.8' r='10.5' data-tip='CART&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 14.7 s&#10;Datasets: 70' aria-label='CART&#10;Improvement over Random: 11.1%&#10;Selection time per 1k features: 14.7 s&#10;Datasets: 70'/><circle class='sa-hit' cx='470.6' cy='165.9' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 12.8 s&#10;Datasets: 70' aria-label='GainRatio&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 12.8 s&#10;Datasets: 70'/><circle class='sa-hit' cx='797.8' cy='167.4' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 2.42e+04 s&#10;Datasets: 70' aria-label='(R)ReliefF&#10;Improvement over Random: 9.2%&#10;Selection time per 1k features: 2.42e+04 s&#10;Datasets: 70'/><circle class='sa-hit' cx='812.3' cy='195.4' r='10.5' data-tip='SFS&#10;Improvement over Random: 7.8%&#10;Selection time per 1k features: 3.39e+04 s&#10;Datasets: 70' aria-label='SFS&#10;Improvement over Random: 7.8%&#10;Selection time per 1k features: 3.39e+04 s&#10;Datasets: 70'/><circle class='sa-hit' cx='486.2' cy='195.7' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 7.8%&#10;Selection time per 1k features: 18.3 s&#10;Datasets: 70' aria-label='MarkovBlanket&#10;Improvement over Random: 7.8%&#10;Selection time per 1k features: 18.3 s&#10;Datasets: 70'/><circle class='sa-hit' cx='740.3' cy='220.0' r='10.5' data-tip='LOCO&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 6.43e+03 s&#10;Datasets: 70' aria-label='LOCO&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 6.43e+03 s&#10;Datasets: 70'/><circle class='sa-hit' cx='797.4' cy='234.4' r='10.5' data-tip='SBE&#10;Improvement over Random: 5.8%&#10;Selection time per 1k features: 2.41e+04 s&#10;Datasets: 70' aria-label='SBE&#10;Improvement over Random: 5.8%&#10;Selection time per 1k features: 2.41e+04 s&#10;Datasets: 70'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 70' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 70'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,15.8586,40.579,70,True
37
  ElasticNet,13.651,2100.4728,70,False
38
  MI,12.4516,46.4375,70,False
data/subsets/tasks_all/models_linearmodel/budgets_4/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1220.0'>1220<span class='sub'>+53 / −48</span></td><td class='num' data-sort='10.6'>10.6</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+48 / −44</span></td><td class='num' data-sort='8.0'>8.0</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1187.0'>1187<span class='sub'>+48 / −39</span></td><td class='num' data-sort='9.5'>9.5</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1182.0'>1182<span class='sub'>+43 / −42</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1176.0'>1176<span class='sub'>+46 / −47</span></td><td class='num' data-sort='9.5'>9.5</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1170.0'>1170<span class='sub'>+54 / −46</span></td><td class='num' data-sort='6.8'>6.8</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+49 / −45</span></td><td class='num' data-sort='6.9'>6.9</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1138.0'>1138<span class='sub'>+52 / −45</span></td><td class='num' data-sort='6.5'>6.5</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='13.4'>13.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1121.0'>1121<span class='sub'>+49 / −49</span></td><td class='num' data-sort='7.4'>7.4</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1114.0'>1114<span class='sub'>+49 / −53</span></td><td class='num' data-sort='7.4'>7.4</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1111.0'>1111<span class='sub'>+45 / −53</span></td><td class='num' data-sort='3.3'>3.3</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='1035.1'>1,035.1</td><td class='num' data-sort='41.3'>41.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1102.0'>1102<span class='sub'>+46 / −46</span></td><td class='num' data-sort='4.0'>4.0</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='3042.8'>3,042.8</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1089.0'>1089<span class='sub'>+45 / −46</span></td><td class='num' data-sort='3.6'>3.6</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='234.7'>234.7</td><td class='num' data-sort='18.3'>18.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1042.0'>1042<span class='sub'>+56 / −66</span></td><td class='num' data-sort='-39154926337779.6'>-39,154,926,337,779.6</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+52 / −59</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1220.0,1172.0,1273.0,10.6,0.597,0.507,0.2,0.0
37
  2,F-Test,Non-search,Correlation,1189.0,1145.0,1237.0,8.0,0.562,0.689,0.0,0.0
38
  3,ElasticNet,Non-search,Regularization,1187.0,1148.0,1235.0,9.5,0.308,0.568,6.2,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1220.0'>1220<span class='sub'>+53 / −48</span></td><td class='num' data-sort='10.6'>10.6</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+48 / −44</span></td><td class='num' data-sort='8.0'>8.0</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1187.0'>1187<span class='sub'>+48 / −39</span></td><td class='num' data-sort='9.5'>9.5</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1182.0'>1182<span class='sub'>+43 / −42</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1176.0'>1176<span class='sub'>+46 / −47</span></td><td class='num' data-sort='9.5'>9.5</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1170.0'>1170<span class='sub'>+54 / −46</span></td><td class='num' data-sort='6.8'>6.8</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+49 / −45</span></td><td class='num' data-sort='6.9'>6.9</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1138.0'>1138<span class='sub'>+52 / −45</span></td><td class='num' data-sort='6.5'>6.5</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='13.4'>13.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1121.0'>1121<span class='sub'>+49 / −49</span></td><td class='num' data-sort='7.4'>7.4</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1114.0'>1114<span class='sub'>+49 / −53</span></td><td class='num' data-sort='7.4'>7.4</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1111.0'>1111<span class='sub'>+45 / −53</span></td><td class='num' data-sort='3.3'>3.3</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='1035.1'>1,035.1</td><td class='num' data-sort='41.3'>41.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1102.0'>1102<span class='sub'>+46 / −46</span></td><td class='num' data-sort='4.0'>4.0</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='3042.8'>3,042.8</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1089.0'>1089<span class='sub'>+45 / −46</span></td><td class='num' data-sort='3.6'>3.6</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='234.7'>234.7</td><td class='num' data-sort='18.3'>18.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1042.0'>1042<span class='sub'>+56 / −66</span></td><td class='num' data-sort='-39154926337779.6'>-39,154,926,337,779.6</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+52 / −59</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1220.0,1172.0,1273.0,10.6,0.597,0.507,0.2,0.0
39
  2,F-Test,Non-search,Correlation,1189.0,1145.0,1237.0,8.0,0.562,0.689,0.0,0.0
40
  3,ElasticNet,Non-search,Regularization,1187.0,1148.0,1235.0,9.5,0.308,0.568,6.2,0.0
data/subsets/tasks_all/models_linearmodel/budgets_4/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='159.1' x2='159.1' y1='16.2' y2='368.0'/><text class='sa-text' x='159.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='258.9' x2='258.9' y1='16.2' y2='368.0'/><text class='sa-text' x='258.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='358.6' x2='358.6' y1='16.2' y2='368.0'/><text class='sa-text' x='358.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='458.3' x2='458.3' y1='16.2' y2='368.0'/><text class='sa-text' x='458.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='558.1' x2='558.1' y1='16.2' y2='368.0'/><text class='sa-text' x='558.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.8' x2='657.8' y1='16.2' y2='368.0'/><text class='sa-text' x='657.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.5' x2='757.5' y1='16.2' y2='368.0'/><text class='sa-text' x='757.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='291.5' y2='291.5'/><text class='sa-text' x='57.9' y='297.5' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='231.1' y2='231.1'/><text class='sa-text' x='57.9' y='237.1' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='170.6' y2='170.6'/><text class='sa-text' x='57.9' y='176.6' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='110.1' y2='110.1'/><text class='sa-text' x='57.9' y='116.1' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='49.6' y2='49.6'/><text class='sa-text' x='57.9' y='55.6' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 340.4,109.6 516.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='688.7' cy='64.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='563.6' cy='65.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='340.4' cy='109.6' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='524.0' cy='123.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.7' cy='127.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='483.4' cy='127.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.3' cy='143.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='687.2' cy='147.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.4' cy='154.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='231.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.5' cy='243.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='779.5' cy='251.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.8' y1='32.2' x2='394.6' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='394.6' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='688.7' y1='64.4' x2='768.0' y2='76.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='768.0' y='76.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='563.6' y1='65.1' x2='603.3' y2='50.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='603.3' y='50.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='340.4' y1='109.6' x2='380.1' y2='95.3' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='380.1' y='95.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='524.0' y1='123.8' x2='644.7' y2='123.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='644.7' y='123.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='468.7' y1='127.0' x2='510.2' y2='88.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='510.2' y='88.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='501.9' y='131.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='474.3' y1='143.8' x2='499.4' y2='169.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='499.4' y='169.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='687.2' y1='147.5' x2='651.9' y2='161.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='651.9' y='161.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><text class='sa-dim sa-halo' x='796.9' y='168.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='231.0' x2='828.6' y2='201.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='201.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='737.5' y1='243.4' x2='773.4' y2='236.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='773.4' y='236.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='779.5' y1='251.6' x2='805.9' y2='267.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='805.9' y='267.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 10.6%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 10.6%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='688.7' cy='64.4' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 9.5%&#10;Selection time per 1k features: 2.04e+03 s&#10;Datasets: 71' aria-label='ElasticNet&#10;Improvement over Random: 9.5%&#10;Selection time per 1k features: 2.04e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='563.6' cy='65.1' r='10.5' data-tip='mRMR&#10;Improvement over Random: 9.5%&#10;Selection time per 1k features: 114 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 9.5%&#10;Selection time per 1k features: 114 s&#10;Datasets: 71'/><circle class='sa-hit' cx='340.4' cy='109.6' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 8.0%&#10;Selection time per 1k features: 0.657 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 8.0%&#10;Selection time per 1k features: 0.657 s&#10;Datasets: 71'/><circle class='sa-hit' cx='524.0' cy='123.8' r='10.5' data-tip='MI&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 45.5 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 45.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='468.7' cy='127.0' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 7.4%, 17.9 s&#10;Next to CART: 6.9%, 14.4 s' aria-label='GainRatio&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 7.4%, 17.9 s&#10;Next to CART: 6.9%, 14.4 s'/><circle class='sa-hit' cx='483.4' cy='127.0' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 17.9 s&#10;Datasets: 71&#10;Next to GainRatio: 7.4%, 12.7 s&#10;Next to CART: 6.9%, 14.4 s' aria-label='MarkovBlanket&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 17.9 s&#10;Datasets: 71&#10;Next to GainRatio: 7.4%, 12.7 s&#10;Next to CART: 6.9%, 14.4 s'/><circle class='sa-hit' cx='474.3' cy='143.8' r='10.5' data-tip='CART&#10;Improvement over Random: 6.9%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71&#10;Next to GainRatio: 7.4%, 12.7 s&#10;Next to MarkovBlanket: 7.4%, 17.9 s' aria-label='CART&#10;Improvement over Random: 6.9%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71&#10;Next to GainRatio: 7.4%, 12.7 s&#10;Next to MarkovBlanket: 7.4%, 17.9 s'/><circle class='sa-hit' cx='687.2' cy='147.5' r='10.5' data-tip='Lasso&#10;Improvement over Random: 6.8%&#10;Selection time per 1k features: 1.97e+03 s&#10;Datasets: 71' aria-label='Lasso&#10;Improvement over Random: 6.8%&#10;Selection time per 1k features: 1.97e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.4' cy='154.8' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 2.4e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 2.4e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='231.0' r='10.5' data-tip='SFS&#10;Improvement over Random: 4.0%&#10;Selection time per 1k features: 3.54e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 4.0%&#10;Selection time per 1k features: 3.54e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='737.5' cy='243.4' r='10.5' data-tip='LOCO&#10;Improvement over Random: 3.6%&#10;Selection time per 1k features: 6.3e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 3.6%&#10;Selection time per 1k features: 6.3e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='779.5' cy='251.6' r='10.5' data-tip='SBE&#10;Improvement over Random: 3.3%&#10;Selection time per 1k features: 1.66e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 3.3%&#10;Selection time per 1k features: 1.66e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,10.5744,38.5313,71,True
37
  ElasticNet,9.5109,2038.5127,71,False
38
  mRMR,9.488,113.6585,71,False
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='159.1' x2='159.1' y1='16.2' y2='368.0'/><text class='sa-text' x='159.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='258.9' x2='258.9' y1='16.2' y2='368.0'/><text class='sa-text' x='258.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='358.6' x2='358.6' y1='16.2' y2='368.0'/><text class='sa-text' x='358.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='458.3' x2='458.3' y1='16.2' y2='368.0'/><text class='sa-text' x='458.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='558.1' x2='558.1' y1='16.2' y2='368.0'/><text class='sa-text' x='558.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.8' x2='657.8' y1='16.2' y2='368.0'/><text class='sa-text' x='657.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.5' x2='757.5' y1='16.2' y2='368.0'/><text class='sa-text' x='757.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='291.5' y2='291.5'/><text class='sa-text' x='57.9' y='297.5' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='231.1' y2='231.1'/><text class='sa-text' x='57.9' y='237.1' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='170.6' y2='170.6'/><text class='sa-text' x='57.9' y='176.6' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='110.1' y2='110.1'/><text class='sa-text' x='57.9' y='116.1' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='49.6' y2='49.6'/><text class='sa-text' x='57.9' y='55.6' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 340.4,109.6 516.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='688.7' cy='64.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='563.6' cy='65.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='340.4' cy='109.6' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='524.0' cy='123.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.7' cy='127.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='483.4' cy='127.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.3' cy='143.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='687.2' cy='147.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.4' cy='154.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='231.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.5' cy='243.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='779.5' cy='251.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.8' y1='32.2' x2='392.7' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='392.7' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='688.7' y1='64.4' x2='767.4' y2='75.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='767.4' y='75.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='563.6' y1='65.1' x2='603.3' y2='50.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='603.3' y='50.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='340.4' y1='109.6' x2='380.1' y2='95.3' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='380.1' y='95.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='524.0' y1='123.8' x2='644.0' y2='122.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='644.0' y='122.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='468.7' y1='127.0' x2='510.9' y2='88.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='510.9' y='88.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='501.3' y='130.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='474.3' y1='143.8' x2='499.4' y2='169.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='499.4' y='169.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='687.2' y1='147.5' x2='653.2' y2='161.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='653.2' y='161.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><text class='sa-dim sa-halo' x='796.9' y='169.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='231.0' x2='828.6' y2='203.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='203.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='737.5' y1='243.4' x2='773.4' y2='237.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='773.4' y='237.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='779.5' y1='251.6' x2='806.6' y2='265.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='806.6' y='265.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 10.6%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 10.6%&#10;Selection time per 1k features: 38.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='688.7' cy='64.4' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 9.5%&#10;Selection time per 1k features: 2.04e+03 s&#10;Datasets: 71' aria-label='ElasticNet&#10;Improvement over Random: 9.5%&#10;Selection time per 1k features: 2.04e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='563.6' cy='65.1' r='10.5' data-tip='mRMR&#10;Improvement over Random: 9.5%&#10;Selection time per 1k features: 114 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 9.5%&#10;Selection time per 1k features: 114 s&#10;Datasets: 71'/><circle class='sa-hit' cx='340.4' cy='109.6' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 8.0%&#10;Selection time per 1k features: 0.657 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 8.0%&#10;Selection time per 1k features: 0.657 s&#10;Datasets: 71'/><circle class='sa-hit' cx='524.0' cy='123.8' r='10.5' data-tip='MI&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 45.5 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 7.5%&#10;Selection time per 1k features: 45.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='468.7' cy='127.0' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 7.4%, 17.9 s&#10;Next to CART: 6.9%, 14.4 s' aria-label='GainRatio&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 7.4%, 17.9 s&#10;Next to CART: 6.9%, 14.4 s'/><circle class='sa-hit' cx='483.4' cy='127.0' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 17.9 s&#10;Datasets: 71&#10;Next to GainRatio: 7.4%, 12.7 s&#10;Next to CART: 6.9%, 14.4 s' aria-label='MarkovBlanket&#10;Improvement over Random: 7.4%&#10;Selection time per 1k features: 17.9 s&#10;Datasets: 71&#10;Next to GainRatio: 7.4%, 12.7 s&#10;Next to CART: 6.9%, 14.4 s'/><circle class='sa-hit' cx='474.3' cy='143.8' r='10.5' data-tip='CART&#10;Improvement over Random: 6.9%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71&#10;Next to GainRatio: 7.4%, 12.7 s&#10;Next to MarkovBlanket: 7.4%, 17.9 s' aria-label='CART&#10;Improvement over Random: 6.9%&#10;Selection time per 1k features: 14.4 s&#10;Datasets: 71&#10;Next to GainRatio: 7.4%, 12.7 s&#10;Next to MarkovBlanket: 7.4%, 17.9 s'/><circle class='sa-hit' cx='687.2' cy='147.5' r='10.5' data-tip='Lasso&#10;Improvement over Random: 6.8%&#10;Selection time per 1k features: 1.97e+03 s&#10;Datasets: 71' aria-label='Lasso&#10;Improvement over Random: 6.8%&#10;Selection time per 1k features: 1.97e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.4' cy='154.8' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 2.4e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 6.5%&#10;Selection time per 1k features: 2.4e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='231.0' r='10.5' data-tip='SFS&#10;Improvement over Random: 4.0%&#10;Selection time per 1k features: 3.54e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 4.0%&#10;Selection time per 1k features: 3.54e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='737.5' cy='243.4' r='10.5' data-tip='LOCO&#10;Improvement over Random: 3.6%&#10;Selection time per 1k features: 6.3e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 3.6%&#10;Selection time per 1k features: 6.3e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='779.5' cy='251.6' r='10.5' data-tip='SBE&#10;Improvement over Random: 3.3%&#10;Selection time per 1k features: 1.66e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 3.3%&#10;Selection time per 1k features: 1.66e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,10.5744,38.5313,71,True
37
  ElasticNet,9.5109,2038.5127,71,False
38
  mRMR,9.488,113.6585,71,False
data/subsets/tasks_all/models_linearmodel/budgets_all/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1286.0'>1286<span class='sub'>+43 / −39</span></td><td class='num' data-sort='22.6'>22.6</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1282.0'>1282<span class='sub'>+44 / −36</span></td><td class='num' data-sort='24.6'>24.6</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1259.0'>1259<span class='sub'>+43 / −41</span></td><td class='num' data-sort='21.7'>21.7</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1257.0'>1257<span class='sub'>+32 / −30</span></td><td class='num' data-sort='23.2'>23.2</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1246.0'>1246<span class='sub'>+32 / −32</span></td><td class='num' data-sort='22.9'>22.9</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1232.0'>1232<span class='sub'>+41 / −36</span></td><td class='num' data-sort='22.0'>22.0</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1213.0'>1213<span class='sub'>+34 / −34</span></td><td class='num' data-sort='21.8'>21.8</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1196.0'>1196<span class='sub'>+33 / −32</span></td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+33 / −31</span></td><td class='num' data-sort='18.4'>18.4</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+35 / −36</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2071.2'>2,071.2</td><td class='num' data-sort='43.6'>43.6</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+33 / −35</span></td><td class='num' data-sort='16.3'>16.3</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1135.0'>1135<span class='sub'>+33 / −36</span></td><td class='num' data-sort='12.0'>12.0</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='7.2'>7.2</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1130.0'>1130<span class='sub'>+34 / −40</span></td><td class='num' data-sort='8.4'>8.4</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2561.3'>2,561.3</td><td class='num' data-sort='46.7'>46.7</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+36 / −39</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='994.0'>994<span class='sub'>+46 / −54</span></td><td class='num' data-sort='-14.4'>-14.4</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,ElasticNet,Non-search,Regularization,1286.0,1247.0,1329.0,22.6,0.447,0.76,6.2,0.0
37
  2,RFImportance,Non-search,Tree,1282.0,1246.0,1326.0,24.6,0.643,0.669,0.2,0.0
38
  3,Lasso,Non-search,Regularization,1259.0,1218.0,1302.0,21.7,0.362,0.73,1.9,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1286.0'>1286<span class='sub'>+43 / −39</span></td><td class='num' data-sort='22.6'>22.6</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1282.0'>1282<span class='sub'>+44 / −36</span></td><td class='num' data-sort='24.6'>24.6</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1259.0'>1259<span class='sub'>+43 / −41</span></td><td class='num' data-sort='21.7'>21.7</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1257.0'>1257<span class='sub'>+32 / −30</span></td><td class='num' data-sort='23.2'>23.2</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1246.0'>1246<span class='sub'>+32 / −32</span></td><td class='num' data-sort='22.9'>22.9</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1232.0'>1232<span class='sub'>+41 / −36</span></td><td class='num' data-sort='22.0'>22.0</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1213.0'>1213<span class='sub'>+34 / −34</span></td><td class='num' data-sort='21.8'>21.8</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1196.0'>1196<span class='sub'>+33 / −32</span></td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+33 / −31</span></td><td class='num' data-sort='18.4'>18.4</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+35 / −36</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2071.2'>2,071.2</td><td class='num' data-sort='43.6'>43.6</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+33 / −35</span></td><td class='num' data-sort='16.3'>16.3</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1135.0'>1135<span class='sub'>+33 / −36</span></td><td class='num' data-sort='12.0'>12.0</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='7.2'>7.2</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1130.0'>1130<span class='sub'>+34 / −40</span></td><td class='num' data-sort='8.4'>8.4</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2561.3'>2,561.3</td><td class='num' data-sort='46.7'>46.7</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+36 / −39</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='994.0'>994<span class='sub'>+46 / −54</span></td><td class='num' data-sort='-14.4'>-14.4</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,ElasticNet,Non-search,Regularization,1286.0,1247.0,1329.0,22.6,0.447,0.76,6.2,0.0
39
  2,RFImportance,Non-search,Tree,1282.0,1246.0,1326.0,24.6,0.643,0.669,0.2,0.0
40
  3,Lasso,Non-search,Regularization,1259.0,1218.0,1302.0,21.7,0.362,0.73,1.9,0.0
data/subsets/tasks_all/models_linearmodel/budgets_all/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='161.4' x2='161.4' y1='16.2' y2='368.0'/><text class='sa-text' x='161.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='262.3' x2='262.3' y1='16.2' y2='368.0'/><text class='sa-text' x='262.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='363.2' x2='363.2' y1='16.2' y2='368.0'/><text class='sa-text' x='363.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='464.1' x2='464.1' y1='16.2' y2='368.0'/><text class='sa-text' x='464.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='565.0' x2='565.0' y1='16.2' y2='368.0'/><text class='sa-text' x='565.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='665.9' x2='665.9' y1='16.2' y2='368.0'/><text class='sa-text' x='665.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='766.7' x2='766.7' y1='16.2' y2='368.0'/><text class='sa-text' x='766.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='286.9' y2='286.9'/><text class='sa-text' x='57.9' y='292.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='221.8' y2='221.8'/><text class='sa-text' x='57.9' y='227.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='156.7' y2='156.7'/><text class='sa-text' x='57.9' y='162.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='91.6' y2='91.6'/><text class='sa-text' x='57.9' y='97.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='26.5' y2='26.5'/><text class='sa-text' x='57.9' y='32.5' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 344.2,54.5 523.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='523.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='530.9' cy='50.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='344.2' cy='54.5' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='698.4' cy='58.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='480.3' cy='66.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='534.6' cy='68.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='696.5' cy='69.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='491.8' cy='104.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.6' cy='112.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='121.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='804.9' cy='140.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='746.8' cy='196.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='811.4' cy='242.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='523.5' y1='32.2' x2='642.5' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='642.5' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='530.9' y1='50.5' x2='638.0' y2='56.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='638.0' y='56.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='344.2' y1='54.5' x2='294.4' y2='62.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='294.4' y='62.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='698.4' y1='58.4' x2='777.5' y2='52.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='777.5' y='52.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='466.5' y='51.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='534.6' y1='68.0' x2='569.2' y2='81.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='569.2' y='81.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='696.5' y1='69.7' x2='734.7' y2='83.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='734.7' y='83.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='491.8' y1='104.4' x2='376.5' y2='91.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='376.5' y='91.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='474.6' y1='112.9' x2='545.7' y2='127.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='545.7' y='127.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='825.5' y='107.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='804.9' y1='140.3' x2='739.6' y2='154.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='739.6' y='154.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='746.8' y1='196.3' x2='714.0' y2='210.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='714.0' y='210.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='811.4' y1='242.8' x2='786.9' y2='256.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='786.9' y='256.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='523.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 24.6%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71&#10;Next to MI: 23.2%, 46 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 24.6%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71&#10;Next to MI: 23.2%, 46 s'/><circle class='sa-hit' cx='530.9' cy='50.5' r='10.5' data-tip='MI&#10;Improvement over Random: 23.2%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71&#10;Next to RFImportance: 24.6%, 38.8 s&#10;Next to mRMR: 21.8%, 50 s' aria-label='MI&#10;Improvement over Random: 23.2%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71&#10;Next to RFImportance: 24.6%, 38.8 s&#10;Next to mRMR: 21.8%, 50 s'/><circle class='sa-hit' cx='344.2' cy='54.5' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 22.9%&#10;Selection time per 1k features: 0.647 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 22.9%&#10;Selection time per 1k features: 0.647 s&#10;Datasets: 71'/><circle class='sa-hit' cx='698.4' cy='58.4' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 22.6%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 71&#10;Next to Lasso: 21.7%, 2.01e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 22.6%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 71&#10;Next to Lasso: 21.7%, 2.01e+03 s'/><circle class='sa-hit' cx='480.3' cy='66.1' r='10.5' data-tip='CART&#10;Improvement over Random: 22.0%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71' aria-label='CART&#10;Improvement over Random: 22.0%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='534.6' cy='68.0' r='10.5' data-tip='mRMR&#10;Improvement over Random: 21.8%&#10;Selection time per 1k features: 50 s&#10;Datasets: 71&#10;Next to MI: 23.2%, 46 s' aria-label='mRMR&#10;Improvement over Random: 21.8%&#10;Selection time per 1k features: 50 s&#10;Datasets: 71&#10;Next to MI: 23.2%, 46 s'/><circle class='sa-hit' cx='696.5' cy='69.7' r='10.5' data-tip='Lasso&#10;Improvement over Random: 21.7%&#10;Selection time per 1k features: 2.01e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 22.6%, 2.1e+03 s' aria-label='Lasso&#10;Improvement over Random: 21.7%&#10;Selection time per 1k features: 2.01e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 22.6%, 2.1e+03 s'/><circle class='sa-hit' cx='491.8' cy='104.4' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 19.0%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71&#10;Next to GainRatio: 18.4%, 12.7 s' aria-label='MarkovBlanket&#10;Improvement over Random: 19.0%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71&#10;Next to GainRatio: 18.4%, 12.7 s'/><circle class='sa-hit' cx='474.6' cy='112.9' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 18.4%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 19.0%, 18.8 s' aria-label='GainRatio&#10;Improvement over Random: 18.4%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 19.0%, 18.8 s'/><circle class='sa-hit' cx='812.3' cy='121.4' r='10.5' data-tip='SFS&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 2.83e+04 s&#10;Datasets: 71&#10;Next to (R)ReliefF: 16.3%, 2.39e+04 s' aria-label='SFS&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 2.83e+04 s&#10;Datasets: 71&#10;Next to (R)ReliefF: 16.3%, 2.39e+04 s'/><circle class='sa-hit' cx='804.9' cy='140.3' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 16.3%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71&#10;Next to SFS: 17.7%, 2.83e+04 s' aria-label='(R)ReliefF&#10;Improvement over Random: 16.3%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71&#10;Next to SFS: 17.7%, 2.83e+04 s'/><circle class='sa-hit' cx='746.8' cy='196.3' r='10.5' data-tip='LOCO&#10;Improvement over Random: 12.0%&#10;Selection time per 1k features: 6.34e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 12.0%&#10;Selection time per 1k features: 6.34e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='811.4' cy='242.8' r='10.5' data-tip='SBE&#10;Improvement over Random: 8.4%&#10;Selection time per 1k features: 2.77e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 8.4%&#10;Selection time per 1k features: 2.77e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,24.5664,38.8189,71,True
37
  MI,23.1632,45.9696,71,False
38
  F-Test,22.8523,0.6474,71,True
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='161.4' x2='161.4' y1='16.2' y2='368.0'/><text class='sa-text' x='161.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='262.3' x2='262.3' y1='16.2' y2='368.0'/><text class='sa-text' x='262.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='363.2' x2='363.2' y1='16.2' y2='368.0'/><text class='sa-text' x='363.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='464.1' x2='464.1' y1='16.2' y2='368.0'/><text class='sa-text' x='464.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='565.0' x2='565.0' y1='16.2' y2='368.0'/><text class='sa-text' x='565.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='665.9' x2='665.9' y1='16.2' y2='368.0'/><text class='sa-text' x='665.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='766.7' x2='766.7' y1='16.2' y2='368.0'/><text class='sa-text' x='766.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='286.9' y2='286.9'/><text class='sa-text' x='57.9' y='292.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='221.8' y2='221.8'/><text class='sa-text' x='57.9' y='227.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='156.7' y2='156.7'/><text class='sa-text' x='57.9' y='162.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='91.6' y2='91.6'/><text class='sa-text' x='57.9' y='97.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='26.5' y2='26.5'/><text class='sa-text' x='57.9' y='32.5' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 344.2,54.5 523.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='523.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='530.9' cy='50.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='344.2' cy='54.5' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='698.4' cy='58.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='480.3' cy='66.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='534.6' cy='68.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='696.5' cy='69.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='491.8' cy='104.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.6' cy='112.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='121.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='804.9' cy='140.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='746.8' cy='196.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='811.4' cy='242.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='523.5' y1='32.2' x2='639.4' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='639.4' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='530.9' y1='50.5' x2='632.9' y2='57.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='632.9' y='57.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='344.2' y1='54.5' x2='294.4' y2='57.4' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='294.4' y='57.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='698.4' y1='58.4' x2='786.3' y2='54.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='786.3' y='54.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='467.1' y='51.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='534.6' y1='68.0' x2='563.5' y2='82.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='563.5' y='82.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='696.5' y1='69.7' x2='732.2' y2='84.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='732.2' y='84.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='491.8' y1='104.4' x2='375.9' y2='94.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='375.9' y='94.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='474.6' y1='112.9' x2='545.7' y2='127.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='545.7' y='127.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='825.5' y='107.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='804.9' y1='140.3' x2='739.6' y2='154.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='739.6' y='154.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='746.8' y1='196.3' x2='714.0' y2='210.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='714.0' y='210.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='811.4' y1='242.8' x2='786.9' y2='256.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='786.9' y='256.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='523.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 24.6%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71&#10;Next to MI: 23.2%, 46 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 24.6%&#10;Selection time per 1k features: 38.8 s&#10;Datasets: 71&#10;Next to MI: 23.2%, 46 s'/><circle class='sa-hit' cx='530.9' cy='50.5' r='10.5' data-tip='MI&#10;Improvement over Random: 23.2%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71&#10;Next to RFImportance: 24.6%, 38.8 s&#10;Next to mRMR: 21.8%, 50 s' aria-label='MI&#10;Improvement over Random: 23.2%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71&#10;Next to RFImportance: 24.6%, 38.8 s&#10;Next to mRMR: 21.8%, 50 s'/><circle class='sa-hit' cx='344.2' cy='54.5' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 22.9%&#10;Selection time per 1k features: 0.647 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 22.9%&#10;Selection time per 1k features: 0.647 s&#10;Datasets: 71'/><circle class='sa-hit' cx='698.4' cy='58.4' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 22.6%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 71&#10;Next to Lasso: 21.7%, 2.01e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 22.6%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 71&#10;Next to Lasso: 21.7%, 2.01e+03 s'/><circle class='sa-hit' cx='480.3' cy='66.1' r='10.5' data-tip='CART&#10;Improvement over Random: 22.0%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71' aria-label='CART&#10;Improvement over Random: 22.0%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='534.6' cy='68.0' r='10.5' data-tip='mRMR&#10;Improvement over Random: 21.8%&#10;Selection time per 1k features: 50 s&#10;Datasets: 71&#10;Next to MI: 23.2%, 46 s' aria-label='mRMR&#10;Improvement over Random: 21.8%&#10;Selection time per 1k features: 50 s&#10;Datasets: 71&#10;Next to MI: 23.2%, 46 s'/><circle class='sa-hit' cx='696.5' cy='69.7' r='10.5' data-tip='Lasso&#10;Improvement over Random: 21.7%&#10;Selection time per 1k features: 2.01e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 22.6%, 2.1e+03 s' aria-label='Lasso&#10;Improvement over Random: 21.7%&#10;Selection time per 1k features: 2.01e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 22.6%, 2.1e+03 s'/><circle class='sa-hit' cx='491.8' cy='104.4' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 19.0%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71&#10;Next to GainRatio: 18.4%, 12.7 s' aria-label='MarkovBlanket&#10;Improvement over Random: 19.0%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71&#10;Next to GainRatio: 18.4%, 12.7 s'/><circle class='sa-hit' cx='474.6' cy='112.9' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 18.4%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 19.0%, 18.8 s' aria-label='GainRatio&#10;Improvement over Random: 18.4%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 19.0%, 18.8 s'/><circle class='sa-hit' cx='812.3' cy='121.4' r='10.5' data-tip='SFS&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 2.83e+04 s&#10;Datasets: 71&#10;Next to (R)ReliefF: 16.3%, 2.39e+04 s' aria-label='SFS&#10;Improvement over Random: 17.7%&#10;Selection time per 1k features: 2.83e+04 s&#10;Datasets: 71&#10;Next to (R)ReliefF: 16.3%, 2.39e+04 s'/><circle class='sa-hit' cx='804.9' cy='140.3' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 16.3%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71&#10;Next to SFS: 17.7%, 2.83e+04 s' aria-label='(R)ReliefF&#10;Improvement over Random: 16.3%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71&#10;Next to SFS: 17.7%, 2.83e+04 s'/><circle class='sa-hit' cx='746.8' cy='196.3' r='10.5' data-tip='LOCO&#10;Improvement over Random: 12.0%&#10;Selection time per 1k features: 6.34e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 12.0%&#10;Selection time per 1k features: 6.34e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='811.4' cy='242.8' r='10.5' data-tip='SBE&#10;Improvement over Random: 8.4%&#10;Selection time per 1k features: 2.77e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 8.4%&#10;Selection time per 1k features: 2.77e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,24.5664,38.8189,71,True
37
  MI,23.1632,45.9696,71,False
38
  F-Test,22.8523,0.6474,71,True
data/subsets/tasks_all/models_randomforest/budgets_0/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1311.0'>1311<span class='sub'>+63 / −49</span></td><td class='num' data-sort='29.6'>29.6</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1278.0'>1278<span class='sub'>+44 / −44</span></td><td class='num' data-sort='29.8'>29.8</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1265.0'>1265<span class='sub'>+59 / −47</span></td><td class='num' data-sort='27.1'>27.1</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1262.0'>1262<span class='sub'>+48 / −46</span></td><td class='num' data-sort='28.1'>28.1</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1257.0'>1257<span class='sub'>+53 / −51</span></td><td class='num' data-sort='27.1'>27.1</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1245.0'>1245<span class='sub'>+54 / −49</span></td><td class='num' data-sort='23.1'>23.1</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='977.7'>977.7</td><td class='num' data-sort='29.6'>29.6</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1236.0'>1236<span class='sub'>+46 / −46</span></td><td class='num' data-sort='24.2'>24.2</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1234.0'>1234<span class='sub'>+46 / −44</span></td><td class='num' data-sort='24.5'>24.5</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1213.0'>1213<span class='sub'>+48 / −45</span></td><td class='num' data-sort='23.3'>23.3</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1201.0'>1201<span class='sub'>+53 / −54</span></td><td class='num' data-sort='21.5'>21.5</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='12.9'>12.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1201.0'>1201<span class='sub'>+42 / −45</span></td><td class='num' data-sort='23.2'>23.2</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1151.0'>1151<span class='sub'>+51 / −49</span></td><td class='num' data-sort='12.6'>12.6</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3536.1'>3,536.1</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1129.0'>1129<span class='sub'>+45 / −46</span></td><td class='num' data-sort='13.1'>13.1</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='254.9'>254.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1003.0'>1003<span class='sub'>+65 / −81</span></td><td class='num' data-sort='-11.0'>-11.0</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+49 / −60</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1311.0,1262.0,1374.0,29.6,0.628,0.805,0.2,0.0
37
  2,MI,Non-search,Information theory,1278.0,1234.0,1322.0,29.8,0.645,0.898,0.4,0.0
38
  3,CART,Non-search,Tree,1265.0,1218.0,1324.0,27.1,0.484,0.731,0.0,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1311.0'>1311<span class='sub'>+63 / −49</span></td><td class='num' data-sort='29.6'>29.6</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1278.0'>1278<span class='sub'>+44 / −44</span></td><td class='num' data-sort='29.8'>29.8</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1265.0'>1265<span class='sub'>+59 / −47</span></td><td class='num' data-sort='27.1'>27.1</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1262.0'>1262<span class='sub'>+48 / −46</span></td><td class='num' data-sort='28.1'>28.1</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1257.0'>1257<span class='sub'>+53 / −51</span></td><td class='num' data-sort='27.1'>27.1</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1245.0'>1245<span class='sub'>+54 / −49</span></td><td class='num' data-sort='23.1'>23.1</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='977.7'>977.7</td><td class='num' data-sort='29.6'>29.6</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1236.0'>1236<span class='sub'>+46 / −46</span></td><td class='num' data-sort='24.2'>24.2</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1234.0'>1234<span class='sub'>+46 / −44</span></td><td class='num' data-sort='24.5'>24.5</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1213.0'>1213<span class='sub'>+48 / −45</span></td><td class='num' data-sort='23.3'>23.3</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1201.0'>1201<span class='sub'>+53 / −54</span></td><td class='num' data-sort='21.5'>21.5</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='12.9'>12.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1201.0'>1201<span class='sub'>+42 / −45</span></td><td class='num' data-sort='23.2'>23.2</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1151.0'>1151<span class='sub'>+51 / −49</span></td><td class='num' data-sort='12.6'>12.6</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3536.1'>3,536.1</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1129.0'>1129<span class='sub'>+45 / −46</span></td><td class='num' data-sort='13.1'>13.1</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='254.9'>254.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1003.0'>1003<span class='sub'>+65 / −81</span></td><td class='num' data-sort='-11.0'>-11.0</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+49 / −60</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1311.0,1262.0,1374.0,29.6,0.628,0.805,0.2,0.0
39
  2,MI,Non-search,Information theory,1278.0,1234.0,1322.0,29.8,0.645,0.898,0.4,0.0
40
  3,CART,Non-search,Tree,1265.0,1218.0,1324.0,27.1,0.484,0.731,0.0,0.0
data/subsets/tasks_all/models_randomforest/budgets_0/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='160.7' x2='160.7' y1='16.2' y2='368.0'/><text class='sa-text' x='160.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='260.2' x2='260.2' y1='16.2' y2='368.0'/><text class='sa-text' x='260.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='359.8' x2='359.8' y1='16.2' y2='368.0'/><text class='sa-text' x='359.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='459.3' x2='459.3' y1='16.2' y2='368.0'/><text class='sa-text' x='459.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='558.8' x2='558.8' y1='16.2' y2='368.0'/><text class='sa-text' x='558.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='658.4' x2='658.4' y1='16.2' y2='368.0'/><text class='sa-text' x='658.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.9' x2='757.9' y1='16.2' y2='368.0'/><text class='sa-text' x='757.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='298.4' y2='298.4'/><text class='sa-text' x='58.0' y='304.4' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='244.7' y2='244.7'/><text class='sa-text' x='58.0' y='250.7' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='191.1' y2='191.1'/><text class='sa-text' x='58.0' y='197.1' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='137.4' y2='137.4'/><text class='sa-text' x='58.0' y='143.4' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='83.7' y2='83.7'/><text class='sa-text' x='58.0' y='89.7' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='30.1' y2='30.1'/><text class='sa-text' x='58.0' y='36.1' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 339.8,50.4 517.8,34.2 526.1,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='526.1' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='517.8' cy='34.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='339.8' cy='50.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='475.2' cy='61.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='486.8' cy='61.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.0' cy='89.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='469.6' cy='92.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='480.2' cy='102.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='687.3' cy='102.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='781.8' cy='104.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.5' cy='120.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='738.9' cy='211.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='217.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='526.1' y1='32.2' x2='423.4' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='423.4' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>MI</text><line class='sa-ink' x1='517.8' y1='34.2' x2='417.2' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='417.2' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='339.8' y1='50.4' x2='257.9' y2='56.7' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='257.9' y='56.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='475.2' y1='61.2' x2='596.1' y2='48.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='596.1' y='48.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='486.8' y1='61.3' x2='590.7' y2='80.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='590.7' y='80.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='691.0' y1='89.6' x2='750.8' y2='54.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='750.8' y='54.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='469.6' y1='92.5' x2='389.7' y2='78.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='389.7' y='78.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='480.2' y1='102.3' x2='524.9' y2='116.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='524.9' y='116.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='687.3' y1='102.9' x2='653.6' y2='117.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='653.6' y='117.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='781.8' y1='104.2' x2='806.9' y2='89.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='806.9' y='89.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><text class='sa-dim sa-halo' x='797.0' y='135.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='738.9' y1='211.9' x2='762.2' y2='197.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='762.2' y='197.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='231.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='526.1' cy='32.2' r='10.5' data-tip='MI (on the Pareto front)&#10;Improvement over Random: 29.8%&#10;Selection time per 1k features: 46.9 s&#10;Datasets: 71&#10;Next to RFImportance: 29.6%, 38.7 s' aria-label='MI (on the Pareto front)&#10;Improvement over Random: 29.8%&#10;Selection time per 1k features: 46.9 s&#10;Datasets: 71&#10;Next to RFImportance: 29.6%, 38.7 s'/><circle class='sa-hit' cx='517.8' cy='34.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.6%&#10;Selection time per 1k features: 38.7 s&#10;Datasets: 71&#10;Next to MI: 29.8%, 46.9 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.6%&#10;Selection time per 1k features: 38.7 s&#10;Datasets: 71&#10;Next to MI: 29.8%, 46.9 s'/><circle class='sa-hit' cx='339.8' cy='50.4' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 28.1%&#10;Selection time per 1k features: 0.63 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 28.1%&#10;Selection time per 1k features: 0.63 s&#10;Datasets: 71'/><circle class='sa-hit' cx='475.2' cy='61.2' r='10.5' data-tip='CART&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 27.1%, 18.9 s' aria-label='CART&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 27.1%, 18.9 s'/><circle class='sa-hit' cx='486.8' cy='61.3' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to CART: 27.1%, 14.5 s' aria-label='MarkovBlanket&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to CART: 27.1%, 14.5 s'/><circle class='sa-hit' cx='691.0' cy='89.6' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 24.4%&#10;Selection time per 1k features: 2.12e+03 s&#10;Datasets: 71&#10;Next to Lasso: 23.2%, 1.95e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 24.4%&#10;Selection time per 1k features: 2.12e+03 s&#10;Datasets: 71&#10;Next to Lasso: 23.2%, 1.95e+03 s'/><circle class='sa-hit' cx='469.6' cy='92.5' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 24.2%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to mRMR: 23.3%, 16.2 s' aria-label='GainRatio&#10;Improvement over Random: 24.2%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to mRMR: 23.3%, 16.2 s'/><circle class='sa-hit' cx='480.2' cy='102.3' r='10.5' data-tip='mRMR&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 16.2 s&#10;Datasets: 71&#10;Next to GainRatio: 24.2%, 12.7 s' aria-label='mRMR&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 16.2 s&#10;Datasets: 71&#10;Next to GainRatio: 24.2%, 12.7 s'/><circle class='sa-hit' cx='687.3' cy='102.9' r='10.5' data-tip='Lasso&#10;Improvement over Random: 23.2%&#10;Selection time per 1k features: 1.95e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 24.4%, 2.12e+03 s' aria-label='Lasso&#10;Improvement over Random: 23.2%&#10;Selection time per 1k features: 1.95e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 24.4%, 2.12e+03 s'/><circle class='sa-hit' cx='781.8' cy='104.2' r='10.5' data-tip='SFS&#10;Improvement over Random: 23.1%&#10;Selection time per 1k features: 1.74e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 23.1%&#10;Selection time per 1k features: 1.74e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.5' cy='120.8' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 21.5%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 21.5%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='738.9' cy='211.9' r='10.5' data-tip='LOCO&#10;Improvement over Random: 13.1%&#10;Selection time per 1k features: 6.44e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 13.1%&#10;Selection time per 1k features: 6.44e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='217.2' r='10.5' data-tip='SBE&#10;Improvement over Random: 12.6%&#10;Selection time per 1k features: 3.52e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 12.6%&#10;Selection time per 1k features: 3.52e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  MI,29.8025,46.9055,71,True
37
  RFImportance,29.6115,38.7025,71,True
38
  F-Test,28.1089,0.63,71,True
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='160.7' x2='160.7' y1='16.2' y2='368.0'/><text class='sa-text' x='160.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='260.2' x2='260.2' y1='16.2' y2='368.0'/><text class='sa-text' x='260.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='359.8' x2='359.8' y1='16.2' y2='368.0'/><text class='sa-text' x='359.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='459.3' x2='459.3' y1='16.2' y2='368.0'/><text class='sa-text' x='459.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='558.8' x2='558.8' y1='16.2' y2='368.0'/><text class='sa-text' x='558.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='658.4' x2='658.4' y1='16.2' y2='368.0'/><text class='sa-text' x='658.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.9' x2='757.9' y1='16.2' y2='368.0'/><text class='sa-text' x='757.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='298.4' y2='298.4'/><text class='sa-text' x='58.0' y='304.4' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='244.7' y2='244.7'/><text class='sa-text' x='58.0' y='250.7' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='191.1' y2='191.1'/><text class='sa-text' x='58.0' y='197.1' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='137.4' y2='137.4'/><text class='sa-text' x='58.0' y='143.4' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='83.7' y2='83.7'/><text class='sa-text' x='58.0' y='89.7' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='30.1' y2='30.1'/><text class='sa-text' x='58.0' y='36.1' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 339.8,50.4 517.8,34.2 526.1,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='526.1' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='517.8' cy='34.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='339.8' cy='50.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='475.2' cy='61.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='486.8' cy='61.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.0' cy='89.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='469.6' cy='92.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='480.2' cy='102.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='687.3' cy='102.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='781.8' cy='104.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.5' cy='120.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='738.9' cy='211.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='217.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='526.1' y1='32.2' x2='423.5' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='423.5' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>MI</text><line class='sa-ink' x1='517.8' y1='34.2' x2='417.7' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='417.7' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='339.8' y1='50.4' x2='246.6' y2='59.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='246.6' y='59.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='475.2' y1='61.2' x2='587.3' y2='46.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='587.3' y='46.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='486.8' y1='61.3' x2='590.7' y2='83.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='590.7' y='83.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='691.0' y1='89.6' x2='759.0' y2='46.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='759.0' y='46.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='469.6' y1='92.5' x2='392.2' y2='81.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='392.2' y='81.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='480.2' y1='102.3' x2='525.6' y2='116.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='525.6' y='116.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='687.3' y1='102.9' x2='654.9' y2='116.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='654.9' y='116.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='781.8' y1='104.2' x2='806.9' y2='89.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='806.9' y='89.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><text class='sa-dim sa-halo' x='797.0' y='134.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='738.9' y1='211.9' x2='762.2' y2='197.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='762.2' y='197.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='231.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='526.1' cy='32.2' r='10.5' data-tip='MI (on the Pareto front)&#10;Improvement over Random: 29.8%&#10;Selection time per 1k features: 46.9 s&#10;Datasets: 71&#10;Next to RFImportance: 29.6%, 38.7 s' aria-label='MI (on the Pareto front)&#10;Improvement over Random: 29.8%&#10;Selection time per 1k features: 46.9 s&#10;Datasets: 71&#10;Next to RFImportance: 29.6%, 38.7 s'/><circle class='sa-hit' cx='517.8' cy='34.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.6%&#10;Selection time per 1k features: 38.7 s&#10;Datasets: 71&#10;Next to MI: 29.8%, 46.9 s' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 29.6%&#10;Selection time per 1k features: 38.7 s&#10;Datasets: 71&#10;Next to MI: 29.8%, 46.9 s'/><circle class='sa-hit' cx='339.8' cy='50.4' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 28.1%&#10;Selection time per 1k features: 0.63 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 28.1%&#10;Selection time per 1k features: 0.63 s&#10;Datasets: 71'/><circle class='sa-hit' cx='475.2' cy='61.2' r='10.5' data-tip='CART&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 27.1%, 18.9 s' aria-label='CART&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 14.5 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 27.1%, 18.9 s'/><circle class='sa-hit' cx='486.8' cy='61.3' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to CART: 27.1%, 14.5 s' aria-label='MarkovBlanket&#10;Improvement over Random: 27.1%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to CART: 27.1%, 14.5 s'/><circle class='sa-hit' cx='691.0' cy='89.6' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 24.4%&#10;Selection time per 1k features: 2.12e+03 s&#10;Datasets: 71&#10;Next to Lasso: 23.2%, 1.95e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 24.4%&#10;Selection time per 1k features: 2.12e+03 s&#10;Datasets: 71&#10;Next to Lasso: 23.2%, 1.95e+03 s'/><circle class='sa-hit' cx='469.6' cy='92.5' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 24.2%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to mRMR: 23.3%, 16.2 s' aria-label='GainRatio&#10;Improvement over Random: 24.2%&#10;Selection time per 1k features: 12.7 s&#10;Datasets: 71&#10;Next to mRMR: 23.3%, 16.2 s'/><circle class='sa-hit' cx='480.2' cy='102.3' r='10.5' data-tip='mRMR&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 16.2 s&#10;Datasets: 71&#10;Next to GainRatio: 24.2%, 12.7 s' aria-label='mRMR&#10;Improvement over Random: 23.3%&#10;Selection time per 1k features: 16.2 s&#10;Datasets: 71&#10;Next to GainRatio: 24.2%, 12.7 s'/><circle class='sa-hit' cx='687.3' cy='102.9' r='10.5' data-tip='Lasso&#10;Improvement over Random: 23.2%&#10;Selection time per 1k features: 1.95e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 24.4%, 2.12e+03 s' aria-label='Lasso&#10;Improvement over Random: 23.2%&#10;Selection time per 1k features: 1.95e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 24.4%, 2.12e+03 s'/><circle class='sa-hit' cx='781.8' cy='104.2' r='10.5' data-tip='SFS&#10;Improvement over Random: 23.1%&#10;Selection time per 1k features: 1.74e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 23.1%&#10;Selection time per 1k features: 1.74e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='795.5' cy='120.8' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 21.5%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 21.5%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='738.9' cy='211.9' r='10.5' data-tip='LOCO&#10;Improvement over Random: 13.1%&#10;Selection time per 1k features: 6.44e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 13.1%&#10;Selection time per 1k features: 6.44e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='217.2' r='10.5' data-tip='SBE&#10;Improvement over Random: 12.6%&#10;Selection time per 1k features: 3.52e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 12.6%&#10;Selection time per 1k features: 3.52e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  MI,29.8025,46.9055,71,True
37
  RFImportance,29.6115,38.7025,71,True
38
  F-Test,28.1089,0.63,71,True
data/subsets/tasks_all/models_randomforest/budgets_1/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1370.0'>1370<span class='sub'>+47 / −47</span></td><td class='num' data-sort='30.7'>30.7</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1316.0'>1316<span class='sub'>+47 / −42</span></td><td class='num' data-sort='28.2'>28.2</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1310.0'>1310<span class='sub'>+52 / −50</span></td><td class='num' data-sort='26.0'>26.0</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='6.6'>6.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1296.0'>1296<span class='sub'>+49 / −51</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1293.0'>1293<span class='sub'>+40 / −37</span></td><td class='num' data-sort='28.5'>28.5</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1290.0'>1290<span class='sub'>+47 / −40</span></td><td class='num' data-sort='26.3'>26.3</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1272.0'>1272<span class='sub'>+45 / −45</span></td><td class='num' data-sort='26.7'>26.7</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1266.0'>1266<span class='sub'>+50 / −47</span></td><td class='num' data-sort='24.1'>24.1</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1254.0'>1254<span class='sub'>+45 / −43</span></td><td class='num' data-sort='24.1'>24.1</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='13.0'>13.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1243.0'>1243<span class='sub'>+57 / −55</span></td><td class='num' data-sort='18.1'>18.1</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3225.7'>3,225.7</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1228.0'>1228<span class='sub'>+52 / −50</span></td><td class='num' data-sort='23.4'>23.4</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1224.0'>1224<span class='sub'>+50 / −53</span></td><td class='num' data-sort='21.0'>21.0</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1665.2'>1,665.2</td><td class='num' data-sort='45.1'>45.1</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+49 / −50</span></td><td class='num' data-sort='16.2'>16.2</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='247.1'>247.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1078.0'>1078<span class='sub'>+71 / −78</span></td><td class='num' data-sort='-16.6'>-16.6</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+47 / −58</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1370.0,1323.0,1417.0,30.7,0.653,0.741,0.2,0.0
37
  2,CART,Non-search,Tree,1316.0,1274.0,1363.0,28.2,0.456,0.638,0.1,0.0
38
  3,ElasticNet,Non-search,Regularization,1310.0,1260.0,1362.0,26.0,0.519,0.873,6.6,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1370.0'>1370<span class='sub'>+47 / −47</span></td><td class='num' data-sort='30.7'>30.7</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1316.0'>1316<span class='sub'>+47 / −42</span></td><td class='num' data-sort='28.2'>28.2</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1310.0'>1310<span class='sub'>+52 / −50</span></td><td class='num' data-sort='26.0'>26.0</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='6.6'>6.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1296.0'>1296<span class='sub'>+49 / −51</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1293.0'>1293<span class='sub'>+40 / −37</span></td><td class='num' data-sort='28.5'>28.5</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1290.0'>1290<span class='sub'>+47 / −40</span></td><td class='num' data-sort='26.3'>26.3</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1272.0'>1272<span class='sub'>+45 / −45</span></td><td class='num' data-sort='26.7'>26.7</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1266.0'>1266<span class='sub'>+50 / −47</span></td><td class='num' data-sort='24.1'>24.1</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1254.0'>1254<span class='sub'>+45 / −43</span></td><td class='num' data-sort='24.1'>24.1</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='13.0'>13.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1243.0'>1243<span class='sub'>+57 / −55</span></td><td class='num' data-sort='18.1'>18.1</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3225.7'>3,225.7</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1228.0'>1228<span class='sub'>+52 / −50</span></td><td class='num' data-sort='23.4'>23.4</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1224.0'>1224<span class='sub'>+50 / −53</span></td><td class='num' data-sort='21.0'>21.0</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1665.2'>1,665.2</td><td class='num' data-sort='45.1'>45.1</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+49 / −50</span></td><td class='num' data-sort='16.2'>16.2</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='247.1'>247.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1078.0'>1078<span class='sub'>+71 / −78</span></td><td class='num' data-sort='-16.6'>-16.6</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+47 / −58</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1370.0,1323.0,1417.0,30.7,0.653,0.741,0.2,0.0
39
  2,CART,Non-search,Tree,1316.0,1274.0,1363.0,28.2,0.456,0.638,0.1,0.0
40
  3,ElasticNet,Non-search,Regularization,1310.0,1260.0,1362.0,26.0,0.519,0.873,6.6,0.0
data/subsets/tasks_all/models_randomforest/budgets_1/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='162.6' x2='162.6' y1='16.2' y2='368.0'/><text class='sa-text' x='162.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='262.3' x2='262.3' y1='16.2' y2='368.0'/><text class='sa-text' x='262.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.1' x2='362.1' y1='16.2' y2='368.0'/><text class='sa-text' x='362.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='461.9' x2='461.9' y1='16.2' y2='368.0'/><text class='sa-text' x='461.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='561.6' x2='561.6' y1='16.2' y2='368.0'/><text class='sa-text' x='561.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='661.4' x2='661.4' y1='16.2' y2='368.0'/><text class='sa-text' x='661.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.2' x2='761.2' y1='16.2' y2='368.0'/><text class='sa-text' x='761.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='299.9' y2='299.9'/><text class='sa-text' x='58.0' y='305.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='247.8' y2='247.8'/><text class='sa-text' x='58.0' y='253.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='195.6' y2='195.6'/><text class='sa-text' x='58.0' y='201.6' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='143.5' y2='143.5'/><text class='sa-text' x='58.0' y='149.5' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='91.3' y2='91.3'/><text class='sa-text' x='58.0' y='97.3' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='39.2' y2='39.2'/><text class='sa-text' x='58.0' y='45.2' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 341.6,73.4 477.3,57.9 520.0,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='520.0' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='528.2' cy='54.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='477.3' cy='57.9' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='341.6' cy='73.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='489.3' cy='77.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='693.6' cy='80.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.2' cy='81.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='473.3' cy='100.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='799.0' cy='100.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='495.0' cy='108.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='801.5' cy='132.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='163.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='741.2' cy='182.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='520.0' y1='32.2' x2='644.7' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='644.7' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-dim sa-halo' x='546.4' y='57.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><text class='sa-text sa-halo' x='473.5' y='43.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='341.6' y1='73.4' x2='369.3' y2='58.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='369.3' y='58.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='489.3' y1='77.7' x2='367.8' y2='87.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='367.8' y='87.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='693.6' y1='80.5' x2='784.1' y2='76.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='784.1' y='76.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='691.2' y1='81.7' x2='660.7' y2='61.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='660.7' y='61.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='473.3' y1='100.4' x2='438.1' y2='125.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='438.1' y='125.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='799.0' y1='100.8' x2='735.2' y2='114.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='735.2' y='114.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='495.0' y1='108.4' x2='548.5' y2='94.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='548.5' y='94.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='801.5' y1='132.6' x2='773.2' y2='144.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='773.2' y='144.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><text class='sa-dim sa-halo' x='827.3' y='177.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='741.2' y1='182.7' x2='707.2' y2='197.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='707.2' y='197.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='520.0' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 30.7%&#10;Selection time per 1k features: 38.3 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 30.7%&#10;Selection time per 1k features: 38.3 s&#10;Datasets: 71'/><circle class='sa-hit' cx='528.2' cy='54.8' r='10.5' data-tip='MI&#10;Improvement over Random: 28.5%&#10;Selection time per 1k features: 46.2 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 28.5%&#10;Selection time per 1k features: 46.2 s&#10;Datasets: 71'/><circle class='sa-hit' cx='477.3' cy='57.9' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 28.2%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 28.2%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71'/><circle class='sa-hit' cx='341.6' cy='73.4' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 26.7%&#10;Selection time per 1k features: 0.623 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 26.7%&#10;Selection time per 1k features: 0.623 s&#10;Datasets: 71'/><circle class='sa-hit' cx='489.3' cy='77.7' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 26.3%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71' aria-label='MarkovBlanket&#10;Improvement over Random: 26.3%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71'/><circle class='sa-hit' cx='693.6' cy='80.5' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 26.0%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 71&#10;Next to Lasso: 25.9%, 1.99e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 26.0%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 71&#10;Next to Lasso: 25.9%, 1.99e+03 s'/><circle class='sa-hit' cx='691.2' cy='81.7' r='10.5' data-tip='Lasso&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 26.0%, 2.1e+03 s' aria-label='Lasso&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 26.0%, 2.1e+03 s'/><circle class='sa-hit' cx='473.3' cy='100.4' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 13 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 13 s&#10;Datasets: 71'/><circle class='sa-hit' cx='799.0' cy='100.8' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='495.0' cy='108.4' r='10.5' data-tip='mRMR&#10;Improvement over Random: 23.4%&#10;Selection time per 1k features: 21.5 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 23.4%&#10;Selection time per 1k features: 21.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='801.5' cy='132.6' r='10.5' data-tip='SFS&#10;Improvement over Random: 21.0%&#10;Selection time per 1k features: 2.54e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 21.0%&#10;Selection time per 1k features: 2.54e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='163.4' r='10.5' data-tip='SBE&#10;Improvement over Random: 18.1%&#10;Selection time per 1k features: 3.25e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 18.1%&#10;Selection time per 1k features: 3.25e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='741.2' cy='182.7' r='10.5' data-tip='LOCO&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 6.31e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 6.31e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0025 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0025 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,30.67,38.2716,71,True
37
  MI,28.4977,46.173,71,False
38
  CART,28.2072,14.2793,71,True
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='162.6' x2='162.6' y1='16.2' y2='368.0'/><text class='sa-text' x='162.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='262.3' x2='262.3' y1='16.2' y2='368.0'/><text class='sa-text' x='262.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.1' x2='362.1' y1='16.2' y2='368.0'/><text class='sa-text' x='362.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='461.9' x2='461.9' y1='16.2' y2='368.0'/><text class='sa-text' x='461.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='561.6' x2='561.6' y1='16.2' y2='368.0'/><text class='sa-text' x='561.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='661.4' x2='661.4' y1='16.2' y2='368.0'/><text class='sa-text' x='661.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.2' x2='761.2' y1='16.2' y2='368.0'/><text class='sa-text' x='761.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='299.9' y2='299.9'/><text class='sa-text' x='58.0' y='305.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='247.8' y2='247.8'/><text class='sa-text' x='58.0' y='253.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='195.6' y2='195.6'/><text class='sa-text' x='58.0' y='201.6' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='143.5' y2='143.5'/><text class='sa-text' x='58.0' y='149.5' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='91.3' y2='91.3'/><text class='sa-text' x='58.0' y='97.3' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='39.2' y2='39.2'/><text class='sa-text' x='58.0' y='45.2' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 341.6,73.4 477.3,57.9 520.0,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='520.0' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='528.2' cy='54.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='477.3' cy='57.9' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='341.6' cy='73.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='489.3' cy='77.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='693.6' cy='80.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.2' cy='81.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='473.3' cy='100.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='799.0' cy='100.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='495.0' cy='108.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='801.5' cy='132.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='163.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='741.2' cy='182.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='520.0' y1='32.2' x2='643.4' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='643.4' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-dim sa-halo' x='547.7' y='59.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><text class='sa-text sa-halo' x='473.5' y='43.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='341.6' y1='73.4' x2='364.3' y2='52.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='364.3' y='52.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='489.3' y1='77.7' x2='372.8' y2='92.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='372.8' y='92.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='693.6' y1='80.5' x2='786.0' y2='76.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='786.0' y='76.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='691.2' y1='81.7' x2='659.5' y2='66.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='659.5' y='66.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='473.3' y1='100.4' x2='450.7' y2='122.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='450.7' y='122.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='799.0' y1='100.8' x2='735.2' y2='114.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='735.2' y='114.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='495.0' y1='108.4' x2='550.4' y2='93.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='550.4' y='93.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='801.5' y1='132.6' x2='773.2' y2='144.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='773.2' y='144.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><text class='sa-dim sa-halo' x='827.3' y='177.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='741.2' y1='182.7' x2='707.2' y2='197.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='707.2' y='197.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='520.0' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 30.7%&#10;Selection time per 1k features: 38.3 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 30.7%&#10;Selection time per 1k features: 38.3 s&#10;Datasets: 71'/><circle class='sa-hit' cx='528.2' cy='54.8' r='10.5' data-tip='MI&#10;Improvement over Random: 28.5%&#10;Selection time per 1k features: 46.2 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 28.5%&#10;Selection time per 1k features: 46.2 s&#10;Datasets: 71'/><circle class='sa-hit' cx='477.3' cy='57.9' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 28.2%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 28.2%&#10;Selection time per 1k features: 14.3 s&#10;Datasets: 71'/><circle class='sa-hit' cx='341.6' cy='73.4' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 26.7%&#10;Selection time per 1k features: 0.623 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 26.7%&#10;Selection time per 1k features: 0.623 s&#10;Datasets: 71'/><circle class='sa-hit' cx='489.3' cy='77.7' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 26.3%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71' aria-label='MarkovBlanket&#10;Improvement over Random: 26.3%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71'/><circle class='sa-hit' cx='693.6' cy='80.5' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 26.0%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 71&#10;Next to Lasso: 25.9%, 1.99e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 26.0%&#10;Selection time per 1k features: 2.1e+03 s&#10;Datasets: 71&#10;Next to Lasso: 25.9%, 1.99e+03 s'/><circle class='sa-hit' cx='691.2' cy='81.7' r='10.5' data-tip='Lasso&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 26.0%, 2.1e+03 s' aria-label='Lasso&#10;Improvement over Random: 25.9%&#10;Selection time per 1k features: 1.99e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 26.0%, 2.1e+03 s'/><circle class='sa-hit' cx='473.3' cy='100.4' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 13 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 13 s&#10;Datasets: 71'/><circle class='sa-hit' cx='799.0' cy='100.8' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 24.1%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='495.0' cy='108.4' r='10.5' data-tip='mRMR&#10;Improvement over Random: 23.4%&#10;Selection time per 1k features: 21.5 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 23.4%&#10;Selection time per 1k features: 21.5 s&#10;Datasets: 71'/><circle class='sa-hit' cx='801.5' cy='132.6' r='10.5' data-tip='SFS&#10;Improvement over Random: 21.0%&#10;Selection time per 1k features: 2.54e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 21.0%&#10;Selection time per 1k features: 2.54e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='163.4' r='10.5' data-tip='SBE&#10;Improvement over Random: 18.1%&#10;Selection time per 1k features: 3.25e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 18.1%&#10;Selection time per 1k features: 3.25e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='741.2' cy='182.7' r='10.5' data-tip='LOCO&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 6.31e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 16.2%&#10;Selection time per 1k features: 6.31e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0025 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0025 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,30.67,38.2716,71,True
37
  MI,28.4977,46.173,71,False
38
  CART,28.2072,14.2793,71,True
data/subsets/tasks_all/models_randomforest/budgets_2/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1377.0'>1377<span class='sub'>+50 / −45</span></td><td class='num' data-sort='26.7'>26.7</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1302.0'>1302<span class='sub'>+46 / −43</span></td><td class='num' data-sort='20.2'>20.2</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1283.0'>1283<span class='sub'>+51 / −46</span></td><td class='num' data-sort='21.8'>21.8</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1259.0'>1259<span class='sub'>+52 / −50</span></td><td class='num' data-sort='20.0'>20.0</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+45 / −44</span></td><td class='num' data-sort='19.9'>19.9</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1239.0'>1239<span class='sub'>+47 / −49</span></td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1236.0'>1236<span class='sub'>+43 / −46</span></td><td class='num' data-sort='18.2'>18.2</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1235.0'>1235<span class='sub'>+53 / −45</span></td><td class='num' data-sort='19.3'>19.3</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='13.0'>13.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1204.0'>1204<span class='sub'>+56 / −56</span></td><td class='num' data-sort='10.2'>10.2</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2621.7'>2,621.7</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1190.0'>1190<span class='sub'>+45 / −44</span></td><td class='num' data-sort='15.3'>15.3</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1179.0'>1179<span class='sub'>+51 / −49</span></td><td class='num' data-sort='16.5'>16.5</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1164.0'>1164<span class='sub'>+45 / −52</span></td><td class='num' data-sort='10.1'>10.1</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2242.0'>2,242.0</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1152.0'>1152<span class='sub'>+48 / −48</span></td><td class='num' data-sort='7.9'>7.9</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='239.1'>239.1</td><td class='num' data-sort='6.6'>6.6</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+54 / −72</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='951.0'>951<span class='sub'>+64 / −77</span></td><td class='num' data-sort='-54.7'>-54.7</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1377.0,1332.0,1427.0,26.7,0.676,0.691,0.2,0.0
37
  2,CART,Non-search,Tree,1302.0,1259.0,1348.0,20.2,0.444,0.572,0.0,0.0
38
  3,MI,Non-search,Information theory,1283.0,1237.0,1334.0,21.8,0.683,0.815,0.4,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1377.0'>1377<span class='sub'>+50 / −45</span></td><td class='num' data-sort='26.7'>26.7</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1302.0'>1302<span class='sub'>+46 / −43</span></td><td class='num' data-sort='20.2'>20.2</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1283.0'>1283<span class='sub'>+51 / −46</span></td><td class='num' data-sort='21.8'>21.8</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1259.0'>1259<span class='sub'>+52 / −50</span></td><td class='num' data-sort='20.0'>20.0</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+45 / −44</span></td><td class='num' data-sort='19.9'>19.9</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1239.0'>1239<span class='sub'>+47 / −49</span></td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1236.0'>1236<span class='sub'>+43 / −46</span></td><td class='num' data-sort='18.2'>18.2</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1235.0'>1235<span class='sub'>+53 / −45</span></td><td class='num' data-sort='19.3'>19.3</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='13.0'>13.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1204.0'>1204<span class='sub'>+56 / −56</span></td><td class='num' data-sort='10.2'>10.2</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2621.7'>2,621.7</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1190.0'>1190<span class='sub'>+45 / −44</span></td><td class='num' data-sort='15.3'>15.3</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1179.0'>1179<span class='sub'>+51 / −49</span></td><td class='num' data-sort='16.5'>16.5</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1164.0'>1164<span class='sub'>+45 / −52</span></td><td class='num' data-sort='10.1'>10.1</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2242.0'>2,242.0</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1152.0'>1152<span class='sub'>+48 / −48</span></td><td class='num' data-sort='7.9'>7.9</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='239.1'>239.1</td><td class='num' data-sort='6.6'>6.6</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+54 / −72</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='951.0'>951<span class='sub'>+64 / −77</span></td><td class='num' data-sort='-54.7'>-54.7</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1377.0,1332.0,1427.0,26.7,0.676,0.691,0.2,0.0
39
  2,CART,Non-search,Tree,1302.0,1259.0,1348.0,20.2,0.444,0.572,0.0,0.0
40
  3,MI,Non-search,Information theory,1283.0,1237.0,1334.0,21.8,0.683,0.815,0.4,0.0
data/subsets/tasks_all/models_randomforest/budgets_2/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='161.2' x2='161.2' y1='16.2' y2='368.0'/><text class='sa-text' x='161.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='261.7' x2='261.7' y1='16.2' y2='368.0'/><text class='sa-text' x='261.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.1' x2='362.1' y1='16.2' y2='368.0'/><text class='sa-text' x='362.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='462.6' x2='462.6' y1='16.2' y2='368.0'/><text class='sa-text' x='462.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='563.0' x2='563.0' y1='16.2' y2='368.0'/><text class='sa-text' x='563.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.5' x2='663.5' y1='16.2' y2='368.0'/><text class='sa-text' x='663.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='764.0' x2='764.0' y1='16.2' y2='368.0'/><text class='sa-text' x='764.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='292.1' y2='292.1'/><text class='sa-text' x='57.9' y='298.1' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='232.1' y2='232.1'/><text class='sa-text' x='57.9' y='238.1' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='172.1' y2='172.1'/><text class='sa-text' x='57.9' y='178.1' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='112.1' y2='112.1'/><text class='sa-text' x='57.9' y='118.1' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='52.2' y2='52.2'/><text class='sa-text' x='57.9' y='58.2' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 349.9,124.6 479.3,109.7 521.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='521.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='529.2' cy='90.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='479.3' cy='109.7' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='695.6' cy='112.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='490.1' cy='113.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='801.9' cy='120.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='349.9' cy='124.6' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='694.6' cy='134.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='514.7' cy='154.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='472.8' cy='168.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='811.6' cy='229.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='231.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='744.4' cy='257.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='521.5' y1='32.2' x2='399.3' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='399.3' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='529.2' y1='90.8' x2='547.5' y2='70.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='547.5' y='70.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='479.3' y1='109.7' x2='515.2' y2='99.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='515.2' y='99.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='695.6' y1='112.5' x2='754.3' y2='96.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='754.3' y='96.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='490.1' y1='113.1' x2='551.2' y2='128.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='551.2' y='128.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='801.9' y1='120.0' x2='755.4' y2='126.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='755.4' y='126.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='349.9' y1='124.6' x2='310.2' y2='138.9' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='310.2' y='138.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='694.6' y1='134.0' x2='730.3' y2='156.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='730.3' y='156.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='514.7' y1='154.4' x2='568.8' y2='157.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='568.8' y='157.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='472.8' y1='168.4' x2='467.2' y2='197.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='467.2' y='197.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='811.6' y1='229.8' x2='827.3' y2='185.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='827.3' y='185.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='812.3' y1='231.0' x2='828.6' y2='215.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='215.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='744.4' y1='257.2' x2='711.6' y2='270.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='711.6' y='270.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='521.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 26.7%&#10;Selection time per 1k features: 38.6 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 26.7%&#10;Selection time per 1k features: 38.6 s&#10;Datasets: 71'/><circle class='sa-hit' cx='529.2' cy='90.8' r='10.5' data-tip='MI&#10;Improvement over Random: 21.8%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 21.8%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71'/><circle class='sa-hit' cx='479.3' cy='109.7' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 20.2%&#10;Selection time per 1k features: 14.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 19.9%, 18.8 s' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 20.2%&#10;Selection time per 1k features: 14.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 19.9%, 18.8 s'/><circle class='sa-hit' cx='695.6' cy='112.5' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 20.0%&#10;Selection time per 1k features: 2.08e+03 s&#10;Datasets: 71' aria-label='ElasticNet&#10;Improvement over Random: 20.0%&#10;Selection time per 1k features: 2.08e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='490.1' cy='113.1' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 19.9%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71&#10;Next to CART: 20.2%, 14.7 s' aria-label='MarkovBlanket&#10;Improvement over Random: 19.9%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71&#10;Next to CART: 20.2%, 14.7 s'/><circle class='sa-hit' cx='801.9' cy='120.0' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 19.3%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 19.3%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='349.9' cy='124.6' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 19.0%&#10;Selection time per 1k features: 0.756 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 19.0%&#10;Selection time per 1k features: 0.756 s&#10;Datasets: 71'/><circle class='sa-hit' cx='694.6' cy='134.0' r='10.5' data-tip='Lasso&#10;Improvement over Random: 18.2%&#10;Selection time per 1k features: 2.04e+03 s&#10;Datasets: 71' aria-label='Lasso&#10;Improvement over Random: 18.2%&#10;Selection time per 1k features: 2.04e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='514.7' cy='154.4' r='10.5' data-tip='mRMR&#10;Improvement over Random: 16.5%&#10;Selection time per 1k features: 33 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 16.5%&#10;Selection time per 1k features: 33 s&#10;Datasets: 71'/><circle class='sa-hit' cx='472.8' cy='168.4' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 12.6 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 12.6 s&#10;Datasets: 71'/><circle class='sa-hit' cx='811.6' cy='229.8' r='10.5' data-tip='SBE&#10;Improvement over Random: 10.2%&#10;Selection time per 1k features: 2.98e+04 s&#10;Datasets: 71&#10;Next to SFS: 10.1%, 3.03e+04 s' aria-label='SBE&#10;Improvement over Random: 10.2%&#10;Selection time per 1k features: 2.98e+04 s&#10;Datasets: 71&#10;Next to SFS: 10.1%, 3.03e+04 s'/><circle class='sa-hit' cx='812.3' cy='231.0' r='10.5' data-tip='SFS&#10;Improvement over Random: 10.1%&#10;Selection time per 1k features: 3.03e+04 s&#10;Datasets: 71&#10;Next to SBE: 10.2%, 2.98e+04 s' aria-label='SFS&#10;Improvement over Random: 10.1%&#10;Selection time per 1k features: 3.03e+04 s&#10;Datasets: 71&#10;Next to SBE: 10.2%, 2.98e+04 s'/><circle class='sa-hit' cx='744.4' cy='257.2' r='10.5' data-tip='LOCO&#10;Improvement over Random: 7.9%&#10;Selection time per 1k features: 6.38e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 7.9%&#10;Selection time per 1k features: 6.38e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,26.6641,38.5521,71,True
37
  MI,21.7783,46.0456,71,False
38
  CART,20.1998,14.6774,71,True
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='161.2' x2='161.2' y1='16.2' y2='368.0'/><text class='sa-text' x='161.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='261.7' x2='261.7' y1='16.2' y2='368.0'/><text class='sa-text' x='261.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.1' x2='362.1' y1='16.2' y2='368.0'/><text class='sa-text' x='362.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='462.6' x2='462.6' y1='16.2' y2='368.0'/><text class='sa-text' x='462.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='563.0' x2='563.0' y1='16.2' y2='368.0'/><text class='sa-text' x='563.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.5' x2='663.5' y1='16.2' y2='368.0'/><text class='sa-text' x='663.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='764.0' x2='764.0' y1='16.2' y2='368.0'/><text class='sa-text' x='764.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='292.1' y2='292.1'/><text class='sa-text' x='57.9' y='298.1' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='232.1' y2='232.1'/><text class='sa-text' x='57.9' y='238.1' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='172.1' y2='172.1'/><text class='sa-text' x='57.9' y='178.1' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='112.1' y2='112.1'/><text class='sa-text' x='57.9' y='118.1' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='52.2' y2='52.2'/><text class='sa-text' x='57.9' y='58.2' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 349.9,124.6 479.3,109.7 521.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='521.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='529.2' cy='90.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='479.3' cy='109.7' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='695.6' cy='112.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='490.1' cy='113.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='801.9' cy='120.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='349.9' cy='124.6' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='694.6' cy='134.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='514.7' cy='154.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='472.8' cy='168.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='811.6' cy='229.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='231.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='744.4' cy='257.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='521.5' y1='32.2' x2='396.8' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='396.8' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='529.2' y1='90.8' x2='547.5' y2='61.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='547.5' y='61.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='479.3' y1='109.7' x2='514.6' y2='100.0' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='514.6' y='100.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='695.6' y1='112.5' x2='754.3' y2='97.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='754.3' y='97.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='490.1' y1='113.1' x2='553.7' y2='131.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='553.7' y='131.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='801.9' y1='120.0' x2='751.6' y2='126.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='751.6' y='126.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='349.9' y1='124.6' x2='310.2' y2='138.9' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='310.2' y='138.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='694.6' y1='134.0' x2='730.3' y2='155.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='730.3' y='155.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='514.7' y1='154.4' x2='565.7' y2='166.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='565.7' y='166.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='472.8' y1='168.4' x2='468.4' y2='196.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='468.4' y='196.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='811.6' y1='229.8' x2='827.3' y2='186.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='827.3' y='186.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='812.3' y1='231.0' x2='828.6' y2='215.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='215.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='744.4' y1='257.2' x2='711.6' y2='270.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='711.6' y='270.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='521.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 26.7%&#10;Selection time per 1k features: 38.6 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 26.7%&#10;Selection time per 1k features: 38.6 s&#10;Datasets: 71'/><circle class='sa-hit' cx='529.2' cy='90.8' r='10.5' data-tip='MI&#10;Improvement over Random: 21.8%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 21.8%&#10;Selection time per 1k features: 46 s&#10;Datasets: 71'/><circle class='sa-hit' cx='479.3' cy='109.7' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 20.2%&#10;Selection time per 1k features: 14.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 19.9%, 18.8 s' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 20.2%&#10;Selection time per 1k features: 14.7 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 19.9%, 18.8 s'/><circle class='sa-hit' cx='695.6' cy='112.5' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 20.0%&#10;Selection time per 1k features: 2.08e+03 s&#10;Datasets: 71' aria-label='ElasticNet&#10;Improvement over Random: 20.0%&#10;Selection time per 1k features: 2.08e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='490.1' cy='113.1' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 19.9%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71&#10;Next to CART: 20.2%, 14.7 s' aria-label='MarkovBlanket&#10;Improvement over Random: 19.9%&#10;Selection time per 1k features: 18.8 s&#10;Datasets: 71&#10;Next to CART: 20.2%, 14.7 s'/><circle class='sa-hit' cx='801.9' cy='120.0' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 19.3%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 19.3%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='349.9' cy='124.6' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 19.0%&#10;Selection time per 1k features: 0.756 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 19.0%&#10;Selection time per 1k features: 0.756 s&#10;Datasets: 71'/><circle class='sa-hit' cx='694.6' cy='134.0' r='10.5' data-tip='Lasso&#10;Improvement over Random: 18.2%&#10;Selection time per 1k features: 2.04e+03 s&#10;Datasets: 71' aria-label='Lasso&#10;Improvement over Random: 18.2%&#10;Selection time per 1k features: 2.04e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='514.7' cy='154.4' r='10.5' data-tip='mRMR&#10;Improvement over Random: 16.5%&#10;Selection time per 1k features: 33 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 16.5%&#10;Selection time per 1k features: 33 s&#10;Datasets: 71'/><circle class='sa-hit' cx='472.8' cy='168.4' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 12.6 s&#10;Datasets: 71' aria-label='GainRatio&#10;Improvement over Random: 15.3%&#10;Selection time per 1k features: 12.6 s&#10;Datasets: 71'/><circle class='sa-hit' cx='811.6' cy='229.8' r='10.5' data-tip='SBE&#10;Improvement over Random: 10.2%&#10;Selection time per 1k features: 2.98e+04 s&#10;Datasets: 71&#10;Next to SFS: 10.1%, 3.03e+04 s' aria-label='SBE&#10;Improvement over Random: 10.2%&#10;Selection time per 1k features: 2.98e+04 s&#10;Datasets: 71&#10;Next to SFS: 10.1%, 3.03e+04 s'/><circle class='sa-hit' cx='812.3' cy='231.0' r='10.5' data-tip='SFS&#10;Improvement over Random: 10.1%&#10;Selection time per 1k features: 3.03e+04 s&#10;Datasets: 71&#10;Next to SBE: 10.2%, 2.98e+04 s' aria-label='SFS&#10;Improvement over Random: 10.1%&#10;Selection time per 1k features: 3.03e+04 s&#10;Datasets: 71&#10;Next to SBE: 10.2%, 2.98e+04 s'/><circle class='sa-hit' cx='744.4' cy='257.2' r='10.5' data-tip='LOCO&#10;Improvement over Random: 7.9%&#10;Selection time per 1k features: 6.38e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 7.9%&#10;Selection time per 1k features: 6.38e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0026 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,26.6641,38.5521,71,True
37
  MI,21.7783,46.0456,71,False
38
  CART,20.1998,14.6774,71,True
data/subsets/tasks_all/models_randomforest/budgets_3/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1320.0'>1320<span class='sub'>+53 / −50</span></td><td class='num' data-sort='19.7'>19.7</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1275.0'>1275<span class='sub'>+54 / −43</span></td><td class='num' data-sort='15.2'>15.2</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1229.0'>1229<span class='sub'>+51 / −54</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1222.0'>1222<span class='sub'>+47 / −41</span></td><td class='num' data-sort='15.5'>15.5</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1206.0'>1206<span class='sub'>+51 / −44</span></td><td class='num' data-sort='14.0'>14.0</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='12.9'>12.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+48 / −48</span></td><td class='num' data-sort='12.9'>12.9</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1188.0'>1188<span class='sub'>+38 / −42</span></td><td class='num' data-sort='12.7'>12.7</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1183.0'>1183<span class='sub'>+45 / −45</span></td><td class='num' data-sort='14.8'>14.8</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1180.0'>1180<span class='sub'>+49 / −48</span></td><td class='num' data-sort='11.9'>11.9</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+54 / −59</span></td><td class='num' data-sort='7.1'>7.1</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1890.2'>1,890.2</td><td class='num' data-sort='45.5'>45.5</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1147.0'>1147<span class='sub'>+46 / −44</span></td><td class='num' data-sort='12.8'>12.8</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1110.0'>1110<span class='sub'>+49 / −49</span></td><td class='num' data-sort='4.1'>4.1</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='252.4'>252.4</td><td class='num' data-sort='11.7'>11.7</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1105.0'>1105<span class='sub'>+44 / −43</span></td><td class='num' data-sort='1.4'>1.4</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2829.3'>2,829.3</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+48 / −57</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='943.0'>943<span class='sub'>+66 / −80</span></td><td class='num' data-sort='-86.6'>-86.6</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1320.0,1270.0,1373.0,19.7,0.661,0.6,0.2,0.0
37
  2,CART,Non-search,Tree,1275.0,1232.0,1329.0,15.2,0.402,0.507,0.1,0.0
38
  3,ElasticNet,Non-search,Regularization,1229.0,1175.0,1280.0,13.7,0.419,0.665,6.1,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1320.0'>1320<span class='sub'>+53 / −50</span></td><td class='num' data-sort='19.7'>19.7</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1275.0'>1275<span class='sub'>+54 / −43</span></td><td class='num' data-sort='15.2'>15.2</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1229.0'>1229<span class='sub'>+51 / −54</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1222.0'>1222<span class='sub'>+47 / −41</span></td><td class='num' data-sort='15.5'>15.5</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1206.0'>1206<span class='sub'>+51 / −44</span></td><td class='num' data-sort='14.0'>14.0</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='12.9'>12.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+48 / −48</span></td><td class='num' data-sort='12.9'>12.9</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1188.0'>1188<span class='sub'>+38 / −42</span></td><td class='num' data-sort='12.7'>12.7</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1183.0'>1183<span class='sub'>+45 / −45</span></td><td class='num' data-sort='14.8'>14.8</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1180.0'>1180<span class='sub'>+49 / −48</span></td><td class='num' data-sort='11.9'>11.9</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+54 / −59</span></td><td class='num' data-sort='7.1'>7.1</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1890.2'>1,890.2</td><td class='num' data-sort='45.5'>45.5</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1147.0'>1147<span class='sub'>+46 / −44</span></td><td class='num' data-sort='12.8'>12.8</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1110.0'>1110<span class='sub'>+49 / −49</span></td><td class='num' data-sort='4.1'>4.1</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='252.4'>252.4</td><td class='num' data-sort='11.7'>11.7</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1105.0'>1105<span class='sub'>+44 / −43</span></td><td class='num' data-sort='1.4'>1.4</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2829.3'>2,829.3</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+48 / −57</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='943.0'>943<span class='sub'>+66 / −80</span></td><td class='num' data-sort='-86.6'>-86.6</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1320.0,1270.0,1373.0,19.7,0.661,0.6,0.2,0.0
39
  2,CART,Non-search,Tree,1275.0,1232.0,1329.0,15.2,0.402,0.507,0.1,0.0
40
  3,ElasticNet,Non-search,Regularization,1229.0,1175.0,1280.0,13.7,0.419,0.665,6.1,0.0
data/subsets/tasks_all/models_randomforest/budgets_3/pareto.html CHANGED
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
- </style><line class='sa-grid' x1='172.0' x2='172.0' y1='16.2' y2='368.0'/><text class='sa-text' x='172.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='270.2' x2='270.2' y1='16.2' y2='368.0'/><text class='sa-text' x='270.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='368.4' x2='368.4' y1='16.2' y2='368.0'/><text class='sa-text' x='368.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='466.7' x2='466.7' y1='16.2' y2='368.0'/><text class='sa-text' x='466.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='564.9' x2='564.9' y1='16.2' y2='368.0'/><text class='sa-text' x='564.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.1' x2='663.1' y1='16.2' y2='368.0'/><text class='sa-text' x='663.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.3' x2='761.3' y1='16.2' y2='368.0'/><text class='sa-text' x='761.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='81.3' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='72.3' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='81.3' x2='847.8' y1='311.5' y2='311.5'/><text class='sa-text' x='72.3' y='317.5' text-anchor='end' font-size='15'>2.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='270.9' y2='270.9'/><text class='sa-text' x='72.3' y='276.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='230.3' y2='230.3'/><text class='sa-text' x='72.3' y='236.3' text-anchor='end' font-size='15'>7.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='189.7' y2='189.7'/><text class='sa-text' x='72.3' y='195.7' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='81.3' x2='847.8' y1='149.1' y2='149.1'/><text class='sa-text' x='72.3' y='155.1' text-anchor='end' font-size='15'>12.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='108.5' y2='108.5'/><text class='sa-text' x='72.3' y='114.5' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='81.3' x2='847.8' y1='67.9' y2='67.9'/><text class='sa-text' x='72.3' y='73.9' text-anchor='end' font-size='15'>17.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='27.4' y2='27.4'/><text class='sa-text' x='72.3' y='33.4' text-anchor='end' font-size='15'>20</text><rect class='sa-frame-line' x='81.3' y='16.2' width='766.5' height='351.8'/><text class='sa-text' x='464.5' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='123.6' y1='51.9' x2='83.3' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='96.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='116.1,352.0 351.0,112.4 482.7,105.4 525.1,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='525.1' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='531.4' cy='100.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='482.7' cy='105.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='351.0' cy='112.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='798.5' cy='125.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='694.9' cy='129.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='692.8' cy='143.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='476.7' cy='144.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='493.7' cy='146.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='547.1' cy='158.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='798.1' cy='236.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='742.0' cy='285.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='813.0' cy='329.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='116.1' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='525.1' y1='32.2' x2='403.3' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='403.3' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='531.4' y1='100.9' x2='549.3' y2='86.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='549.3' y='86.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><text class='sa-text sa-halo' x='484.6' y='91.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='351.0' y1='112.4' x2='300.9' y2='125.0' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='300.9' y='125.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><text class='sa-dim sa-halo' x='797.9' y='110.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='694.9' y1='129.1' x2='637.9' y2='114.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='637.9' y='114.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='692.8' y1='143.1' x2='649.9' y2='151.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='649.9' y='151.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='476.7' y1='144.3' x2='370.1' y2='158.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='370.1' y='158.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='493.7' y1='146.1' x2='511.9' y2='191.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='511.9' y='191.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='547.1' y1='158.4' x2='647.9' y2='185.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='647.9' y='185.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='798.1' y1='236.8' x2='819.7' y2='222.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='819.7' y='222.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='742.0' y1='285.8' x2='708.6' y2='300.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='708.6' y='300.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='813.0' y1='329.8' x2='790.7' y2='343.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='790.7' y='343.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='116.1' y1='352.0' x2='168.9' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='168.9' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='525.1' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 19.7%&#10;Selection time per 1k features: 39.4 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 19.7%&#10;Selection time per 1k features: 39.4 s&#10;Datasets: 71'/><circle class='sa-hit' cx='531.4' cy='100.9' r='10.5' data-tip='MI&#10;Improvement over Random: 15.5%&#10;Selection time per 1k features: 45.6 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 15.5%&#10;Selection time per 1k features: 45.6 s&#10;Datasets: 71'/><circle class='sa-hit' cx='482.7' cy='105.4' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 15.2%&#10;Selection time per 1k features: 14.6 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 15.2%&#10;Selection time per 1k features: 14.6 s&#10;Datasets: 71'/><circle class='sa-hit' cx='351.0' cy='112.4' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 14.8%&#10;Selection time per 1k features: 0.664 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 14.8%&#10;Selection time per 1k features: 0.664 s&#10;Datasets: 71'/><circle class='sa-hit' cx='798.5' cy='125.0' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 14.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 14.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='694.9' cy='129.1' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 2.11e+03 s&#10;Datasets: 71&#10;Next to Lasso: 12.9%, 2e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 2.11e+03 s&#10;Datasets: 71&#10;Next to Lasso: 12.9%, 2e+03 s'/><circle class='sa-hit' cx='692.8' cy='143.1' r='10.5' data-tip='Lasso&#10;Improvement over Random: 12.9%&#10;Selection time per 1k features: 2e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.7%, 2.11e+03 s' aria-label='Lasso&#10;Improvement over Random: 12.9%&#10;Selection time per 1k features: 2e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.7%, 2.11e+03 s'/><circle class='sa-hit' cx='476.7' cy='144.3' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 12.8%&#10;Selection time per 1k features: 12.6 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 12.7%, 18.9 s' aria-label='GainRatio&#10;Improvement over Random: 12.8%&#10;Selection time per 1k features: 12.6 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 12.7%, 18.9 s'/><circle class='sa-hit' cx='493.7' cy='146.1' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 12.7%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 12.8%, 12.6 s' aria-label='MarkovBlanket&#10;Improvement over Random: 12.7%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 12.8%, 12.6 s'/><circle class='sa-hit' cx='547.1' cy='158.4' r='10.5' data-tip='mRMR&#10;Improvement over Random: 11.9%&#10;Selection time per 1k features: 65.9 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 11.9%&#10;Selection time per 1k features: 65.9 s&#10;Datasets: 71'/><circle class='sa-hit' cx='798.1' cy='236.8' r='10.5' data-tip='SBE&#10;Improvement over Random: 7.1%&#10;Selection time per 1k features: 2.37e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 7.1%&#10;Selection time per 1k features: 2.37e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='742.0' cy='285.8' r='10.5' data-tip='LOCO&#10;Improvement over Random: 4.1%&#10;Selection time per 1k features: 6.36e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 4.1%&#10;Selection time per 1k features: 6.36e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='813.0' cy='329.8' r='10.5' data-tip='SFS&#10;Improvement over Random: 1.4%&#10;Selection time per 1k features: 3.35e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 1.4%&#10;Selection time per 1k features: 3.35e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='116.1' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,19.7026,39.3595,71,True
37
  MI,15.4689,45.5885,71,False
38
  CART,15.1945,14.5695,71,True
 
32
  .sa-legend { fill:var(--bg); stroke:var(--line); }
33
  .sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
34
  .sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
35
+ </style><line class='sa-grid' x1='172.0' x2='172.0' y1='16.2' y2='368.0'/><text class='sa-text' x='172.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='270.2' x2='270.2' y1='16.2' y2='368.0'/><text class='sa-text' x='270.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='368.4' x2='368.4' y1='16.2' y2='368.0'/><text class='sa-text' x='368.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='466.7' x2='466.7' y1='16.2' y2='368.0'/><text class='sa-text' x='466.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='564.9' x2='564.9' y1='16.2' y2='368.0'/><text class='sa-text' x='564.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.1' x2='663.1' y1='16.2' y2='368.0'/><text class='sa-text' x='663.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.3' x2='761.3' y1='16.2' y2='368.0'/><text class='sa-text' x='761.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='81.3' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='72.3' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='81.3' x2='847.8' y1='311.5' y2='311.5'/><text class='sa-text' x='72.3' y='317.5' text-anchor='end' font-size='15'>2.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='270.9' y2='270.9'/><text class='sa-text' x='72.3' y='276.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='230.3' y2='230.3'/><text class='sa-text' x='72.3' y='236.3' text-anchor='end' font-size='15'>7.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='189.7' y2='189.7'/><text class='sa-text' x='72.3' y='195.7' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='81.3' x2='847.8' y1='149.1' y2='149.1'/><text class='sa-text' x='72.3' y='155.1' text-anchor='end' font-size='15'>12.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='108.5' y2='108.5'/><text class='sa-text' x='72.3' y='114.5' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='81.3' x2='847.8' y1='67.9' y2='67.9'/><text class='sa-text' x='72.3' y='73.9' text-anchor='end' font-size='15'>17.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='27.4' y2='27.4'/><text class='sa-text' x='72.3' y='33.4' text-anchor='end' font-size='15'>20</text><rect class='sa-frame-line' x='81.3' y='16.2' width='766.5' height='351.8'/><text class='sa-text' x='464.5' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='123.6' y1='51.9' x2='83.3' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='96.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='116.1,352.0 351.0,112.4 482.7,105.4 525.1,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='525.1' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='531.4' cy='100.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='482.7' cy='105.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='351.0' cy='112.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='798.5' cy='125.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='694.9' cy='129.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='692.8' cy='143.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='476.7' cy='144.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='493.7' cy='146.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='547.1' cy='158.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='798.1' cy='236.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='742.0' cy='285.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='813.0' cy='329.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='116.1' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='525.1' y1='32.2' x2='402.7' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='402.7' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='531.4' y1='100.9' x2='549.3' y2='86.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='549.3' y='86.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><text class='sa-text sa-halo' x='484.0' y='91.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='351.0' y1='112.4' x2='301.5' y2='125.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='301.5' y='125.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><text class='sa-dim sa-halo' x='797.9' y='110.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='694.9' y1='129.1' x2='636.7' y2='114.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='636.7' y='114.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='692.8' y1='143.1' x2='648.7' y2='150.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='648.7' y='150.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='476.7' y1='144.3' x2='370.1' y2='158.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='370.1' y='158.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='493.7' y1='146.1' x2='510.7' y2='188.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='510.7' y='188.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='547.1' y1='158.4' x2='647.9' y2='187.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='647.9' y='187.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='798.1' y1='236.8' x2='819.7' y2='222.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='819.7' y='222.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='742.0' y1='285.8' x2='709.9' y2='299.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='709.9' y='299.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='813.0' y1='329.8' x2='790.1' y2='344.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='790.1' y='344.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='116.1' y1='352.0' x2='168.9' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='168.9' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='525.1' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front)&#10;Improvement over Random: 19.7%&#10;Selection time per 1k features: 39.4 s&#10;Datasets: 71' aria-label='RFImportance (on the Pareto front)&#10;Improvement over Random: 19.7%&#10;Selection time per 1k features: 39.4 s&#10;Datasets: 71'/><circle class='sa-hit' cx='531.4' cy='100.9' r='10.5' data-tip='MI&#10;Improvement over Random: 15.5%&#10;Selection time per 1k features: 45.6 s&#10;Datasets: 71' aria-label='MI&#10;Improvement over Random: 15.5%&#10;Selection time per 1k features: 45.6 s&#10;Datasets: 71'/><circle class='sa-hit' cx='482.7' cy='105.4' r='10.5' data-tip='CART (on the Pareto front)&#10;Improvement over Random: 15.2%&#10;Selection time per 1k features: 14.6 s&#10;Datasets: 71' aria-label='CART (on the Pareto front)&#10;Improvement over Random: 15.2%&#10;Selection time per 1k features: 14.6 s&#10;Datasets: 71'/><circle class='sa-hit' cx='351.0' cy='112.4' r='10.5' data-tip='F-Test (on the Pareto front)&#10;Improvement over Random: 14.8%&#10;Selection time per 1k features: 0.664 s&#10;Datasets: 71' aria-label='F-Test (on the Pareto front)&#10;Improvement over Random: 14.8%&#10;Selection time per 1k features: 0.664 s&#10;Datasets: 71'/><circle class='sa-hit' cx='798.5' cy='125.0' r='10.5' data-tip='(R)ReliefF&#10;Improvement over Random: 14.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71' aria-label='(R)ReliefF&#10;Improvement over Random: 14.0%&#10;Selection time per 1k features: 2.39e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='694.9' cy='129.1' r='10.5' data-tip='ElasticNet&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 2.11e+03 s&#10;Datasets: 71&#10;Next to Lasso: 12.9%, 2e+03 s' aria-label='ElasticNet&#10;Improvement over Random: 13.7%&#10;Selection time per 1k features: 2.11e+03 s&#10;Datasets: 71&#10;Next to Lasso: 12.9%, 2e+03 s'/><circle class='sa-hit' cx='692.8' cy='143.1' r='10.5' data-tip='Lasso&#10;Improvement over Random: 12.9%&#10;Selection time per 1k features: 2e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.7%, 2.11e+03 s' aria-label='Lasso&#10;Improvement over Random: 12.9%&#10;Selection time per 1k features: 2e+03 s&#10;Datasets: 71&#10;Next to ElasticNet: 13.7%, 2.11e+03 s'/><circle class='sa-hit' cx='476.7' cy='144.3' r='10.5' data-tip='GainRatio&#10;Improvement over Random: 12.8%&#10;Selection time per 1k features: 12.6 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 12.7%, 18.9 s' aria-label='GainRatio&#10;Improvement over Random: 12.8%&#10;Selection time per 1k features: 12.6 s&#10;Datasets: 71&#10;Next to MarkovBlanket: 12.7%, 18.9 s'/><circle class='sa-hit' cx='493.7' cy='146.1' r='10.5' data-tip='MarkovBlanket&#10;Improvement over Random: 12.7%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 12.8%, 12.6 s' aria-label='MarkovBlanket&#10;Improvement over Random: 12.7%&#10;Selection time per 1k features: 18.9 s&#10;Datasets: 71&#10;Next to GainRatio: 12.8%, 12.6 s'/><circle class='sa-hit' cx='547.1' cy='158.4' r='10.5' data-tip='mRMR&#10;Improvement over Random: 11.9%&#10;Selection time per 1k features: 65.9 s&#10;Datasets: 71' aria-label='mRMR&#10;Improvement over Random: 11.9%&#10;Selection time per 1k features: 65.9 s&#10;Datasets: 71'/><circle class='sa-hit' cx='798.1' cy='236.8' r='10.5' data-tip='SBE&#10;Improvement over Random: 7.1%&#10;Selection time per 1k features: 2.37e+04 s&#10;Datasets: 71' aria-label='SBE&#10;Improvement over Random: 7.1%&#10;Selection time per 1k features: 2.37e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='742.0' cy='285.8' r='10.5' data-tip='LOCO&#10;Improvement over Random: 4.1%&#10;Selection time per 1k features: 6.36e+03 s&#10;Datasets: 71' aria-label='LOCO&#10;Improvement over Random: 4.1%&#10;Selection time per 1k features: 6.36e+03 s&#10;Datasets: 71'/><circle class='sa-hit' cx='813.0' cy='329.8' r='10.5' data-tip='SFS&#10;Improvement over Random: 1.4%&#10;Selection time per 1k features: 3.35e+04 s&#10;Datasets: 71' aria-label='SFS&#10;Improvement over Random: 1.4%&#10;Selection time per 1k features: 3.35e+04 s&#10;Datasets: 71'/><circle class='sa-hit' cx='116.1' cy='352.0' r='10.5' data-tip='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71' aria-label='Random (on the Pareto front)&#10;Improvement over Random: 0.0%&#10;Selection time per 1k features: 0.0027 s&#10;Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
36
  RFImportance,19.7026,39.3595,71,True
37
  MI,15.4689,45.5885,71,False
38
  CART,15.1945,14.5695,71,True
data/subsets/tasks_all/models_randomforest/budgets_4/leaderboard_table.html CHANGED
@@ -24,6 +24,8 @@ th[aria-sort] { color:var(--accent); }
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
 
 
27
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
28
  td.na { color:var(--muted); }
29
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
@@ -32,7 +34,7 @@ td.na { color:var(--muted); }
32
  .pill.cat-baseline { color:var(--muted); }
33
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
34
  td.shaded { font-weight:600; }
35
- </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1265.0'>1265<span class='sub'>+43 / −37</span></td><td class='num' data-sort='11.7'>11.7</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1217.0'>1217<span class='sub'>+46 / −39</span></td><td class='num' data-sort='7.2'>7.2</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+48 / −44</span></td><td class='num' data-sort='8.4'>8.4</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='12.6'>12.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1181.0'>1181<span class='sub'>+41 / −42</span></td><td class='num' data-sort='8.6'>8.6</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1176.0'>1176<span class='sub'>+50 / −49</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+52 / −51</span></td><td class='num' data-sort='10.7'>10.7</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+49 / −49</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1145.0'>1145<span class='sub'>+56 / −53</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1128.0'>1128<span class='sub'>+51 / −47</span></td><td class='num' data-sort='6.0'>6.0</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1120.0'>1120<span class='sub'>+47 / −42</span></td><td class='num' data-sort='4.6'>4.6</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1116.0'>1116<span class='sub'>+47 / −49</span></td><td class='num' data-sort='4.7'>4.7</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='1029.1'>1,029.1</td><td class='num' data-sort='40.8'>40.8</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1092.0'>1092<span class='sub'>+48 / −46</span></td><td class='num' data-sort='3.5'>3.5</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='246.5'>246.5</td><td class='num' data-sort='18.3'>18.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1080.0'>1080<span class='sub'>+43 / −46</span></td><td class='num' data-sort='0.6'>0.6</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='3207.4'>3,207.4</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+50 / −54</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='947.0'>947<span class='sub'>+60 / −71</span></td><td class='num' data-sort='-81.5'>-81.5</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
36
  1,RFImportance,Non-search,Tree,1265.0,1228.0,1308.0,11.7,0.597,0.507,0.2,0.0
37
  2,CART,Non-search,Tree,1217.0,1178.0,1263.0,7.2,0.324,0.482,0.1,0.0
38
  3,(R)ReliefF,Non-search,Distance,1192.0,1148.0,1240.0,8.4,0.612,0.594,12.6,0.0
 
24
  tbody tr:hover { background:var(--soft); }
25
  tbody tr:last-child td { border-bottom:none; }
26
  td.method { font-weight:600; }
27
+ td.method a.paper { color:inherit; text-decoration:none; border-bottom:1px dotted var(--muted); }
28
+ td.method a.paper:hover { color:var(--accent); border-bottom-color:var(--accent); }
29
  td .sub { color:var(--muted); font-size:12px; font-weight:400; margin-left:6px; }
30
  td.na { color:var(--muted); }
31
  .pill { display:inline-block; padding:0 8px; border:1px solid var(--line); border-radius:999px;
 
34
  .pill.cat-baseline { color:var(--muted); }
35
  .medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
36
  td.shaded { font-weight:600; }
37
+ </style></head><body><div class='toolbar'><span class='hint'>Click a column to sort and a method to open its paper. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class='' title='Click a method to open its paper (the one cited in Table 2 of the SelectArena paper).'>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'><a class='paper' href='https://doi.org/10.1023/A:1010933404324' target='_blank' rel='noopener'>RFImportance</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1265.0'>1265<span class='sub'>+43 / −37</span></td><td class='num' data-sort='11.7'>11.7</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'><a class='paper' href='https://doi.org/10.1201/9781315139470' target='_blank' rel='noopener'>CART</a></td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1217.0'>1217<span class='sub'>+46 / −39</span></td><td class='num' data-sort='7.2'>7.2</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='(R)ReliefF'><a class='paper' href='https://doi.org/10.1023/A:1008280620621' target='_blank' rel='noopener'>(R)ReliefF</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+48 / −44</span></td><td class='num' data-sort='8.4'>8.4</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='12.6'>12.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'><a class='paper' href='https://doi.org/10.1109/72.298224' target='_blank' rel='noopener'>MI</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1181.0'>1181<span class='sub'>+41 / −42</span></td><td class='num' data-sort='8.6'>8.6</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='ElasticNet'><a class='paper' href='https://doi.org/10.1111/j.1467-9868.2005.00503.x' target='_blank' rel='noopener'>ElasticNet</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1176.0'>1176<span class='sub'>+50 / −49</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'><a class='paper' href='https://doi.org/10.1016/0169-7439(89)80095-4' target='_blank' rel='noopener'>F-Test</a></td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+52 / −51</span></td><td class='num' data-sort='10.7'>10.7</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'><a class='paper' href='http://ilpubs.stanford.edu:8090/208/' target='_blank' rel='noopener'>MarkovBlanket</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+49 / −49</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'><a class='paper' href='https://doi.org/10.1109/TPAMI.2005.159' target='_blank' rel='noopener'>mRMR</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1145.0'>1145<span class='sub'>+56 / −53</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'><a class='paper' href='https://doi.org/10.1007/BF00116251' target='_blank' rel='noopener'>GainRatio</a></td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1128.0'>1128<span class='sub'>+51 / −47</span></td><td class='num' data-sort='6.0'>6.0</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='Lasso'><a class='paper' href='https://doi.org/10.1111/j.2517-6161.1996.tb02080.x' target='_blank' rel='noopener'>Lasso</a></td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1120.0'>1120<span class='sub'>+47 / −42</span></td><td class='num' data-sort='4.6'>4.6</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'><a class='paper' href='https://doi.org/10.1023/A:1012487302797' target='_blank' rel='noopener'>SBE</a></td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1116.0'>1116<span class='sub'>+47 / −49</span></td><td class='num' data-sort='4.7'>4.7</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='1029.1'>1,029.1</td><td class='num' data-sort='40.8'>40.8</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'><a class='paper' href='https://arxiv.org/abs/1604.04173' target='_blank' rel='noopener'>LOCO</a></td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1092.0'>1092<span class='sub'>+48 / −46</span></td><td class='num' data-sort='3.5'>3.5</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='246.5'>246.5</td><td class='num' data-sort='18.3'>18.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'><a class='paper' href='https://doi.org/10.1109/TSMC.1977.4309803' target='_blank' rel='noopener'>SFS</a></td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1080.0'>1080<span class='sub'>+43 / −46</span></td><td class='num' data-sort='0.6'>0.6</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='3207.4'>3,207.4</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+50 / −54</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'><a class='paper' href='https://proceedings.neurips.cc/paper_files/paper/2005/file/b5b03f06271f8917685d14cea7c6c50a-Paper.pdf' target='_blank' rel='noopener'>LaplacianScore</a></td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='947.0'>947<span class='sub'>+60 / −71</span></td><td class='num' data-sort='-81.5'>-81.5</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
38
  1,RFImportance,Non-search,Tree,1265.0,1228.0,1308.0,11.7,0.597,0.507,0.2,0.0
39
  2,CART,Non-search,Tree,1217.0,1178.0,1263.0,7.2,0.324,0.482,0.1,0.0
40
  3,(R)ReliefF,Non-search,Distance,1192.0,1148.0,1240.0,8.4,0.612,0.594,12.6,0.0