Spaces:
Running
Running
Commit ·
09c8054
1
Parent(s): c54fafc
Leaderboard data: interactive Pareto and lasagna figures
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- data/meta.json +1 -1
- data/subsets/tasks_all/models_all/budgets_0/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_all/budgets_0/pareto.html +1 -1
- data/subsets/tasks_all/models_all/budgets_0/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_all/budgets_1/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_all/budgets_1/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_all/budgets_2/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_all/budgets_2/pareto.html +1 -1
- data/subsets/tasks_all/models_all/budgets_2/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_all/budgets_3/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_all/budgets_3/pareto.html +1 -1
- data/subsets/tasks_all/models_all/budgets_3/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_all/budgets_4/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_all/budgets_4/pareto.html +1 -1
- data/subsets/tasks_all/models_all/budgets_4/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_all/budgets_all/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_all/budgets_all/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_0/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_0/pareto.html +1 -1
- data/subsets/tasks_all/models_lightgbm/budgets_0/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_1/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_1/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_2/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_2/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_3/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_3/pareto.html +1 -1
- data/subsets/tasks_all/models_lightgbm/budgets_3/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_4/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_4/pareto.html +1 -1
- data/subsets/tasks_all/models_lightgbm/budgets_4/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_all/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_lightgbm/budgets_all/pareto.html +1 -1
- data/subsets/tasks_all/models_lightgbm/budgets_all/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_0/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_0/pareto.html +1 -1
- data/subsets/tasks_all/models_linearmodel/budgets_0/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_1/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_1/pareto.html +1 -1
- data/subsets/tasks_all/models_linearmodel/budgets_1/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_2/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_2/pareto.html +1 -1
- data/subsets/tasks_all/models_linearmodel/budgets_2/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_3/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_3/pareto.html +1 -1
- data/subsets/tasks_all/models_linearmodel/budgets_3/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_4/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_4/pareto.html +1 -1
- data/subsets/tasks_all/models_linearmodel/budgets_4/website_leaderboard.csv +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_all/leaderboard_table.html +16 -16
- data/subsets/tasks_all/models_linearmodel/budgets_all/pareto.html +1 -1
data/meta.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
{
|
| 2 |
"benchmark_name": "feature_selection_benchmark_2026_all",
|
| 3 |
-
"generated_at": "2026-11-04
|
| 4 |
"selectarena_version": "0.1.0",
|
| 5 |
"n_results": 79873,
|
| 6 |
"n_datasets": 71,
|
|
|
|
| 1 |
{
|
| 2 |
"benchmark_name": "feature_selection_benchmark_2026_all",
|
| 3 |
+
"generated_at": "2026-11-04 20:17 UTC",
|
| 4 |
"selectarena_version": "0.1.0",
|
| 5 |
"n_results": 79873,
|
| 6 |
"n_datasets": 71,
|
data/subsets/tasks_all/models_all/budgets_0/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1343.0'>1343<span class='sub'>+52 / −46</span></td><td class='num' data-sort='6.29'>6.29</td><td class='num' data-sort='20.0'>20.0</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1315.0'>1315<span class='sub'>+47 / −42</span></td><td class='num' data-sort='6.56'>6.56</td><td class='num' data-sort='22.6'>22.6</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1308.0'>1308<span class='sub'>+44 / −41</span></td><td class='num' data-sort='6.67'>6.67</td><td class='num' data-sort='21.8'>21.8</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1290.0'>1290<span class='sub'>+41 / −40</span></td><td class='num' data-sort='7.11'>7.11</td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1290.0'>1290<span class='sub'>+34 / −35</span></td><td class='num' data-sort='6.95'>6.95</td><td class='num' data-sort='23.0'>23.0</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1284.0'>1284<span class='sub'>+42 / −40</span></td><td class='num' data-sort='6.95'>6.95</td><td class='num' data-sort='24.4'>24.4</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='7.5'>7.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1270.0'>1270<span class='sub'>+44 / −46</span></td><td class='num' data-sort='7.13'>7.13</td><td class='num' data-sort='24.5'>24.5</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='941.9'>941.9</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1255.0'>1255<span class='sub'>+39 / −38</span></td><td class='num' data-sort='7.68'>7.68</td><td class='num' data-sort='24.7'>24.7</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1250.0'>1250<span class='sub'>+36 / −41</span></td><td class='num' data-sort='7.42'>7.42</td><td class='num' data-sort='25.7'>25.7</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='2.2'>2.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1242.0'>1242<span class='sub'>+37 / −39</span></td><td class='num' data-sort='7.98'>7.98</td><td class='num' data-sort='26.5'>26.5</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1206.0'>1206<span class='sub'>+44 / −45</span></td><td class='num' data-sort='8.47'>8.47</td><td class='num' data-sort='28.4'>28.4</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='14.2'>14.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+48 / −47</span></td><td class='num' data-sort='8.7'>8.70</td><td class='num' data-sort='30.3'>30.3</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3579.1'>3,579.1</td><td class='num' data-sort='49.6'>49.6</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1176.0'>1176<span class='sub'>+42 / −41</span></td><td class='num' data-sort='8.54'>8.54</td><td class='num' data-sort='29.9'>29.9</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+46 / −54</span></td><td class='num' data-sort='11.38'>11.38</td><td class='num' data-sort='42.0'>42.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='963.0'>963<span class='sub'>+61 / −72</span></td><td class='num' data-sort='12.16'>12.16</td><td class='num' data-sort='45.3'>45.3</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1343.0,1297.0,1395.0,
|
| 37 |
-
2,CART,Non-search,Tree,1315.0,1273.0,1362.0,
|
| 38 |
-
3,MarkovBlanket,Search,Backward search,1308.0,1267.0,1352.0,
|
| 39 |
-
4,F-Test,Non-search,Correlation,1290.0,1250.0,1331.0,
|
| 40 |
-
5,MI,Non-search,Information theory,1290.0,1255.0,1324.0,
|
| 41 |
-
6,ElasticNet,Non-search,Regularization,1284.0,1244.0,1326.0,
|
| 42 |
-
7,SFS,Search,Forward search,1270.0,1224.0,1314.0,
|
| 43 |
-
8,mRMR,Non-search,Information theory,1255.0,1217.0,1294.0,
|
| 44 |
-
9,Lasso,Non-search,Regularization,1250.0,1209.0,1286.0,
|
| 45 |
-
10,GainRatio,Non-search,Information theory,1242.0,1203.0,1279.0,
|
| 46 |
-
11,(R)ReliefF,Non-search,Distance,1206.0,1161.0,1250.0,
|
| 47 |
-
12,SBE,Search,Backward search,1186.0,1139.0,1234.0,
|
| 48 |
-
13,LOCO,Search,Leave-one-out,1176.0,1135.0,1218.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,946.0,1046.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,963.0,891.0,1024.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1343.0'>1343<span class='sub'>+52 / −46</span></td><td class='num' data-sort='29.9'>29.9</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1315.0'>1315<span class='sub'>+47 / −42</span></td><td class='num' data-sort='27.6'>27.6</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1308.0'>1308<span class='sub'>+44 / −41</span></td><td class='num' data-sort='26.8'>26.8</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1290.0'>1290<span class='sub'>+41 / −40</span></td><td class='num' data-sort='28.3'>28.3</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1290.0'>1290<span class='sub'>+34 / −35</span></td><td class='num' data-sort='28.7'>28.7</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1284.0'>1284<span class='sub'>+42 / −40</span></td><td class='num' data-sort='25.6'>25.6</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='7.5'>7.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1270.0'>1270<span class='sub'>+44 / −46</span></td><td class='num' data-sort='23.7'>23.7</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='941.9'>941.9</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1255.0'>1255<span class='sub'>+39 / −38</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1250.0'>1250<span class='sub'>+36 / −41</span></td><td class='num' data-sort='24.1'>24.1</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='2.2'>2.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1242.0'>1242<span class='sub'>+37 / −39</span></td><td class='num' data-sort='23.6'>23.6</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1206.0'>1206<span class='sub'>+44 / −45</span></td><td class='num' data-sort='20.4'>20.4</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='14.2'>14.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+48 / −47</span></td><td class='num' data-sort='12.5'>12.5</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3579.1'>3,579.1</td><td class='num' data-sort='49.6'>49.6</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1176.0'>1176<span class='sub'>+42 / −41</span></td><td class='num' data-sort='15.3'>15.3</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+46 / −54</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='963.0'>963<span class='sub'>+61 / −72</span></td><td class='num' data-sort='-14.7'>-14.7</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1343.0,1297.0,1395.0,29.9,0.628,0.805,0.2,0.0
|
| 37 |
+
2,CART,Non-search,Tree,1315.0,1273.0,1362.0,27.6,0.484,0.731,0.1,0.0
|
| 38 |
+
3,MarkovBlanket,Search,Backward search,1308.0,1267.0,1352.0,26.8,0.418,0.465,0.2,0.0
|
| 39 |
+
4,F-Test,Non-search,Correlation,1290.0,1250.0,1331.0,28.3,0.725,0.952,0.0,0.0
|
| 40 |
+
5,MI,Non-search,Information theory,1290.0,1255.0,1324.0,28.7,0.645,0.898,0.4,0.0
|
| 41 |
+
6,ElasticNet,Non-search,Regularization,1284.0,1244.0,1326.0,25.6,0.509,0.915,7.5,0.0
|
| 42 |
+
7,SFS,Search,Forward search,1270.0,1224.0,1314.0,23.7,0.368,0.75,941.9,29.1
|
| 43 |
+
8,mRMR,Non-search,Information theory,1255.0,1217.0,1294.0,25.5,0.528,0.779,0.3,0.0
|
| 44 |
+
9,Lasso,Non-search,Regularization,1250.0,1209.0,1286.0,24.1,0.421,0.89,2.2,0.0
|
| 45 |
+
10,GainRatio,Non-search,Information theory,1242.0,1203.0,1279.0,23.6,0.679,0.942,0.2,0.0
|
| 46 |
+
11,(R)ReliefF,Non-search,Distance,1206.0,1161.0,1250.0,20.4,0.642,0.873,14.2,0.0
|
| 47 |
+
12,SBE,Search,Backward search,1186.0,1139.0,1234.0,12.5,0.195,0.621,3579.1,49.6
|
| 48 |
+
13,LOCO,Search,Leave-one-out,1176.0,1135.0,1218.0,15.3,0.299,0.665,242.0,0.0
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,946.0,1046.0,0.0,-0.001,0.493,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,963.0,891.0,1024.0,-14.7,0.588,0.847,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_all/budgets_0/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='96.7' x2='96.7' y1='16.2' y2='368.0'/><text class='sa-text' x='96.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='206.1' x2='206.1' y1='16.2' y2='368.0'/><text class='sa-text' x='206.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='315.6' x2='315.6' y1='16.2' y2='368.0'/><text class='sa-text' x='315.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='425.1' x2='425.1' y1='16.2' y2='368.0'/><text class='sa-text' x='425.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='534.5' x2='534.5' y1='16.2' y2='368.0'/><text class='sa-text' x='534.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='644.0' x2='644.0' y1='16.2' y2='368.0'/><text class='sa-text' x='644.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='753.5' x2='753.5' y1='16.2' y2='368.0'/><text class='sa-text' x='753.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='298.6' y2='298.6'/><text class='sa-text' x='58.0' y='304.6' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='245.2' y2='245.2'/><text class='sa-text' x='58.0' y='251.2' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='191.8' y2='191.8'/><text class='sa-text' x='58.0' y='197.8' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='138.4' y2='138.4'/><text class='sa-text' x='58.0' y='144.4' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='85.0' y2='85.0'/><text class='sa-text' x='58.0' y='91.0' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='31.6' y2='31.6'/><text class='sa-text' x='58.0' y='37.6' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 305.1,49.9 496.6,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='496.6' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='502.9' cy='45.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='305.1' cy='49.9' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='446.4' cy='57.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='461.7' cy='65.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.3' cy='78.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='454.0' cy='79.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='684.5' cy='94.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='778.9' cy='98.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='442.0' cy='100.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.9' cy='133.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='731.5' cy='188.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='218.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='496.6' y1='32.2' x2='587.9' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='587.9' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='502.9' y1='45.6' x2='582.9' y2='53.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='582.9' y='53.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='305.1' y1='49.9' x2='265.5' y2='63.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='265.5' y='63.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='446.4' y1='57.0' x2='411.2' y2='46.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='411.2' y='46.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='461.7' y1='65.6' x2='586.5' y2='93.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='586.5' y='93.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='686.3' y1='78.3' x2='793.3' y2='53.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='793.3' y='53.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='454.0' y1='79.6' x2='553.5' y2='122.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='553.5' y='122.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='684.5' y1='94.4' x2='732.2' y2='121.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='732.2' y='121.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='778.9' y1='98.7' x2='803.5' y2='87.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='803.5' y='87.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='442.0' y1='100.3' x2='411.2' y2='123.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='411.2' y='123.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='795.9' y1='133.8' x2='797.0' y2='160.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='797.0' y='160.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='731.5' y1='188.2' x2='698.7' y2='201.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='698.7' y='201.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='812.3' y1='218.4' x2='787.8' y2='232.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='787.8' y='232.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='496.6' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 29.9% Selection time per 1k features: 45 s Datasets: 71 Next to MI: 28.7%, 51.4 s' aria-label='RFImportance (on the Pareto front) Improvement over Random: 29.9% Selection time per 1k features: 45 s Datasets: 71 Next to MI: 28.7%, 51.4 s'/><circle class='sa-hit' cx='502.9' cy='45.6' r='10.5' data-tip='MI Improvement over Random: 28.7% Selection time per 1k features: 51.4 s Datasets: 71 Next to RFImportance: 29.9%, 45 s' aria-label='MI Improvement over Random: 28.7% Selection time per 1k features: 51.4 s Datasets: 71 Next to RFImportance: 29.9%, 45 s'/><circle class='sa-hit' cx='305.1' cy='49.9' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 28.3% Selection time per 1k features: 0.802 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 28.3% Selection time per 1k features: 0.802 s Datasets: 71'/><circle class='sa-hit' cx='446.4' cy='57.0' r='10.5' data-tip='CART Improvement over Random: 27.6% Selection time per 1k features: 15.7 s Datasets: 71 Next to MarkovBlanket: 26.8%, 21.6 s' aria-label='CART Improvement over Random: 27.6% Selection time per 1k features: 15.7 s Datasets: 71 Next to MarkovBlanket: 26.8%, 21.6 s'/><circle class='sa-hit' cx='461.7' cy='65.6' r='10.5' data-tip='MarkovBlanket Improvement over Random: 26.8% Selection time per 1k features: 21.6 s Datasets: 71 Next to CART: 27.6%, 15.7 s Next to mRMR: 25.5%, 18.4 s' aria-label='MarkovBlanket Improvement over Random: 26.8% Selection time per 1k features: 21.6 s Datasets: 71 Next to CART: 27.6%, 15.7 s Next to mRMR: 25.5%, 18.4 s'/><circle class='sa-hit' cx='686.3' cy='78.3' r='10.5' data-tip='ElasticNet Improvement over Random: 25.6% Selection time per 1k features: 2.43e+03 s Datasets: 71 Next to Lasso: 24.1%, 2.35e+03 s' aria-label='ElasticNet Improvement over Random: 25.6% Selection time per 1k features: 2.43e+03 s Datasets: 71 Next to Lasso: 24.1%, 2.35e+03 s'/><circle class='sa-hit' cx='454.0' cy='79.6' r='10.5' data-tip='mRMR Improvement over Random: 25.5% Selection time per 1k features: 18.4 s Datasets: 71 Next to MarkovBlanket: 26.8%, 21.6 s' aria-label='mRMR Improvement over Random: 25.5% Selection time per 1k features: 18.4 s Datasets: 71 Next to MarkovBlanket: 26.8%, 21.6 s'/><circle class='sa-hit' cx='684.5' cy='94.4' r='10.5' data-tip='Lasso Improvement over Random: 24.1% Selection time per 1k features: 2.35e+03 s Datasets: 71 Next to ElasticNet: 25.6%, 2.43e+03 s' aria-label='Lasso Improvement over Random: 24.1% Selection time per 1k features: 2.35e+03 s Datasets: 71 Next to ElasticNet: 25.6%, 2.43e+03 s'/><circle class='sa-hit' cx='778.9' cy='98.7' r='10.5' data-tip='SFS Improvement over Random: 23.7% Selection time per 1k features: 1.71e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 23.7% Selection time per 1k features: 1.71e+04 s Datasets: 71'/><circle class='sa-hit' cx='442.0' cy='100.3' r='10.5' data-tip='GainRatio Improvement over Random: 23.6% Selection time per 1k features: 14.3 s Datasets: 71' aria-label='GainRatio Improvement over Random: 23.6% Selection time per 1k features: 14.3 s Datasets: 71'/><circle class='sa-hit' cx='795.9' cy='133.8' r='10.5' data-tip='(R)ReliefF Improvement over Random: 20.4% Selection time per 1k features: 2.44e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 20.4% Selection time per 1k features: 2.44e+04 s Datasets: 71'/><circle class='sa-hit' cx='731.5' cy='188.2' r='10.5' data-tip='LOCO Improvement over Random: 15.3% Selection time per 1k features: 6.3e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 15.3% Selection time per 1k features: 6.3e+03 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='218.4' r='10.5' data-tip='SBE Improvement over Random: 12.5% Selection time per 1k features: 3.45e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 12.5% Selection time per 1k features: 3.45e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0113 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0113 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,29.9487,44.9997,71,True
|
| 37 |
MI,28.6934,51.3992,71,False
|
| 38 |
F-Test,28.2875,0.8021,71,True
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='96.7' x2='96.7' y1='16.2' y2='368.0'/><text class='sa-text' x='96.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='206.1' x2='206.1' y1='16.2' y2='368.0'/><text class='sa-text' x='206.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='315.6' x2='315.6' y1='16.2' y2='368.0'/><text class='sa-text' x='315.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='425.1' x2='425.1' y1='16.2' y2='368.0'/><text class='sa-text' x='425.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='534.5' x2='534.5' y1='16.2' y2='368.0'/><text class='sa-text' x='534.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='644.0' x2='644.0' y1='16.2' y2='368.0'/><text class='sa-text' x='644.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='753.5' x2='753.5' y1='16.2' y2='368.0'/><text class='sa-text' x='753.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='298.6' y2='298.6'/><text class='sa-text' x='58.0' y='304.6' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='245.2' y2='245.2'/><text class='sa-text' x='58.0' y='251.2' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='191.8' y2='191.8'/><text class='sa-text' x='58.0' y='197.8' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='138.4' y2='138.4'/><text class='sa-text' x='58.0' y='144.4' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='85.0' y2='85.0'/><text class='sa-text' x='58.0' y='91.0' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='31.6' y2='31.6'/><text class='sa-text' x='58.0' y='37.6' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 305.1,49.9 496.6,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='496.6' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='502.9' cy='45.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='305.1' cy='49.9' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='446.4' cy='57.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='461.7' cy='65.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.3' cy='78.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='454.0' cy='79.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='684.5' cy='94.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='778.9' cy='98.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='442.0' cy='100.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.9' cy='133.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='731.5' cy='188.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='218.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='496.6' y1='32.2' x2='589.1' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='589.1' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='502.9' y1='45.6' x2='590.4' y2='57.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='590.4' y='57.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='305.1' y1='49.9' x2='265.5' y2='63.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='265.5' y='63.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='446.4' y1='57.0' x2='412.4' y2='45.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='412.4' y='45.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='461.7' y1='65.6' x2='572.0' y2='86.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='572.0' y='86.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='686.3' y1='78.3' x2='794.5' y2='58.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='794.5' y='58.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='454.0' y1='79.6' x2='552.2' y2='117.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='552.2' y='117.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='684.5' y1='94.4' x2='723.4' y2='120.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='723.4' y='120.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='778.9' y1='98.7' x2='806.0' y2='86.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='806.0' y='86.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='442.0' y1='100.3' x2='413.7' y2='121.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='413.7' y='121.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='797.0' y='149.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='731.5' y1='188.2' x2='698.7' y2='201.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='698.7' y='201.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='812.3' y1='218.4' x2='787.8' y2='232.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='787.8' y='232.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='496.6' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 29.9% Selection time per 1k features: 45 s Datasets: 71 Next to MI: 28.7%, 51.4 s' aria-label='RFImportance (on the Pareto front) Improvement over Random: 29.9% Selection time per 1k features: 45 s Datasets: 71 Next to MI: 28.7%, 51.4 s'/><circle class='sa-hit' cx='502.9' cy='45.6' r='10.5' data-tip='MI Improvement over Random: 28.7% Selection time per 1k features: 51.4 s Datasets: 71 Next to RFImportance: 29.9%, 45 s' aria-label='MI Improvement over Random: 28.7% Selection time per 1k features: 51.4 s Datasets: 71 Next to RFImportance: 29.9%, 45 s'/><circle class='sa-hit' cx='305.1' cy='49.9' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 28.3% Selection time per 1k features: 0.802 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 28.3% Selection time per 1k features: 0.802 s Datasets: 71'/><circle class='sa-hit' cx='446.4' cy='57.0' r='10.5' data-tip='CART Improvement over Random: 27.6% Selection time per 1k features: 15.7 s Datasets: 71 Next to MarkovBlanket: 26.8%, 21.6 s' aria-label='CART Improvement over Random: 27.6% Selection time per 1k features: 15.7 s Datasets: 71 Next to MarkovBlanket: 26.8%, 21.6 s'/><circle class='sa-hit' cx='461.7' cy='65.6' r='10.5' data-tip='MarkovBlanket Improvement over Random: 26.8% Selection time per 1k features: 21.6 s Datasets: 71 Next to CART: 27.6%, 15.7 s Next to mRMR: 25.5%, 18.4 s' aria-label='MarkovBlanket Improvement over Random: 26.8% Selection time per 1k features: 21.6 s Datasets: 71 Next to CART: 27.6%, 15.7 s Next to mRMR: 25.5%, 18.4 s'/><circle class='sa-hit' cx='686.3' cy='78.3' r='10.5' data-tip='ElasticNet Improvement over Random: 25.6% Selection time per 1k features: 2.43e+03 s Datasets: 71 Next to Lasso: 24.1%, 2.35e+03 s' aria-label='ElasticNet Improvement over Random: 25.6% Selection time per 1k features: 2.43e+03 s Datasets: 71 Next to Lasso: 24.1%, 2.35e+03 s'/><circle class='sa-hit' cx='454.0' cy='79.6' r='10.5' data-tip='mRMR Improvement over Random: 25.5% Selection time per 1k features: 18.4 s Datasets: 71 Next to MarkovBlanket: 26.8%, 21.6 s' aria-label='mRMR Improvement over Random: 25.5% Selection time per 1k features: 18.4 s Datasets: 71 Next to MarkovBlanket: 26.8%, 21.6 s'/><circle class='sa-hit' cx='684.5' cy='94.4' r='10.5' data-tip='Lasso Improvement over Random: 24.1% Selection time per 1k features: 2.35e+03 s Datasets: 71 Next to ElasticNet: 25.6%, 2.43e+03 s' aria-label='Lasso Improvement over Random: 24.1% Selection time per 1k features: 2.35e+03 s Datasets: 71 Next to ElasticNet: 25.6%, 2.43e+03 s'/><circle class='sa-hit' cx='778.9' cy='98.7' r='10.5' data-tip='SFS Improvement over Random: 23.7% Selection time per 1k features: 1.71e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 23.7% Selection time per 1k features: 1.71e+04 s Datasets: 71'/><circle class='sa-hit' cx='442.0' cy='100.3' r='10.5' data-tip='GainRatio Improvement over Random: 23.6% Selection time per 1k features: 14.3 s Datasets: 71' aria-label='GainRatio Improvement over Random: 23.6% Selection time per 1k features: 14.3 s Datasets: 71'/><circle class='sa-hit' cx='795.9' cy='133.8' r='10.5' data-tip='(R)ReliefF Improvement over Random: 20.4% Selection time per 1k features: 2.44e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 20.4% Selection time per 1k features: 2.44e+04 s Datasets: 71'/><circle class='sa-hit' cx='731.5' cy='188.2' r='10.5' data-tip='LOCO Improvement over Random: 15.3% Selection time per 1k features: 6.3e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 15.3% Selection time per 1k features: 6.3e+03 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='218.4' r='10.5' data-tip='SBE Improvement over Random: 12.5% Selection time per 1k features: 3.45e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 12.5% Selection time per 1k features: 3.45e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0113 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0113 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,29.9487,44.9997,71,True
|
| 37 |
MI,28.6934,51.3992,71,False
|
| 38 |
F-Test,28.2875,0.8021,71,True
|
data/subsets/tasks_all/models_all/budgets_0/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1343.0,1297.0,1395.0,6.29,62.2,20.0,,0.628,0.805,0.2,0.0,71,1065
|
| 3 |
-
2,CART,Non-search,Tree,1315.0,1273.0,1362.0,6.56,60.3,22.6,,0.484,0.731,0.1,0.0,71,1065
|
| 4 |
-
3,MarkovBlanket,Search,Backward search,1308.0,1267.0,1352.0,6.67,59.5,21.8,,0.418,0.465,0.2,0.0,71,1065
|
| 5 |
-
4,F-Test,Non-search,Correlation,1290.0,1250.0,1331.0,7.11,56.4,23.8,,0.725,0.952,0.0,0.0,71,1065
|
| 6 |
-
5,MI,Non-search,Information theory,1290.0,1255.0,1324.0,6.95,57.5,23.0,,0.645,0.898,0.4,0.0,71,1065
|
| 7 |
-
6,ElasticNet,Non-search,Regularization,1284.0,1244.0,1326.0,6.95,57.5,24.4,,0.509,0.915,7.5,0.0,71,1065
|
| 8 |
-
7,SFS,Search,Forward search,1270.0,1224.0,1314.0,7.13,56.2,24.5,,0.368,0.75,941.9,29.1,71,1065
|
| 9 |
-
8,mRMR,Non-search,Information theory,1255.0,1217.0,1294.0,7.68,52.3,24.7,,0.528,0.779,0.3,0.0,71,1065
|
| 10 |
-
9,Lasso,Non-search,Regularization,1250.0,1209.0,1286.0,7.42,54.1,25.7,,0.421,0.89,2.2,0.0,71,1065
|
| 11 |
-
10,GainRatio,Non-search,Information theory,1242.0,1203.0,1279.0,7.98,50.1,26.5,,0.679,0.942,0.2,0.0,71,1065
|
| 12 |
-
11,(R)ReliefF,Non-search,Distance,1206.0,1161.0,1250.0,8.47,46.6,28.4,,0.642,0.873,14.2,0.0,71,1065
|
| 13 |
-
12,SBE,Search,Backward search,1186.0,1139.0,1234.0,8.7,45.0,30.3,,0.195,0.621,3579.1,49.6,71,1065
|
| 14 |
-
13,LOCO,Search,Leave-one-out,1176.0,1135.0,1218.0,8.54,46.1,29.9,,0.299,0.665,242.0,0.0,71,1065
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,946.0,1046.0,11.38,25.9,42.0,,-0.001,0.493,0.0,0.0,71,1065
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,963.0,891.0,1024.0,12.16,20.3,45.3,,0.588,0.847,0.2,0.0,71,1065
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1343.0,1297.0,1395.0,6.29,62.2,20.0,29.9,,0.628,0.805,0.2,0.0,71,1065
|
| 3 |
+
2,CART,Non-search,Tree,1315.0,1273.0,1362.0,6.56,60.3,22.6,27.6,,0.484,0.731,0.1,0.0,71,1065
|
| 4 |
+
3,MarkovBlanket,Search,Backward search,1308.0,1267.0,1352.0,6.67,59.5,21.8,26.8,,0.418,0.465,0.2,0.0,71,1065
|
| 5 |
+
4,F-Test,Non-search,Correlation,1290.0,1250.0,1331.0,7.11,56.4,23.8,28.3,,0.725,0.952,0.0,0.0,71,1065
|
| 6 |
+
5,MI,Non-search,Information theory,1290.0,1255.0,1324.0,6.95,57.5,23.0,28.7,,0.645,0.898,0.4,0.0,71,1065
|
| 7 |
+
6,ElasticNet,Non-search,Regularization,1284.0,1244.0,1326.0,6.95,57.5,24.4,25.6,,0.509,0.915,7.5,0.0,71,1065
|
| 8 |
+
7,SFS,Search,Forward search,1270.0,1224.0,1314.0,7.13,56.2,24.5,23.7,,0.368,0.75,941.9,29.1,71,1065
|
| 9 |
+
8,mRMR,Non-search,Information theory,1255.0,1217.0,1294.0,7.68,52.3,24.7,25.5,,0.528,0.779,0.3,0.0,71,1065
|
| 10 |
+
9,Lasso,Non-search,Regularization,1250.0,1209.0,1286.0,7.42,54.1,25.7,24.1,,0.421,0.89,2.2,0.0,71,1065
|
| 11 |
+
10,GainRatio,Non-search,Information theory,1242.0,1203.0,1279.0,7.98,50.1,26.5,23.6,,0.679,0.942,0.2,0.0,71,1065
|
| 12 |
+
11,(R)ReliefF,Non-search,Distance,1206.0,1161.0,1250.0,8.47,46.6,28.4,20.4,,0.642,0.873,14.2,0.0,71,1065
|
| 13 |
+
12,SBE,Search,Backward search,1186.0,1139.0,1234.0,8.7,45.0,30.3,12.5,,0.195,0.621,3579.1,49.6,71,1065
|
| 14 |
+
13,LOCO,Search,Leave-one-out,1176.0,1135.0,1218.0,8.54,46.1,29.9,15.3,,0.299,0.665,242.0,0.0,71,1065
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,946.0,1046.0,11.38,25.9,42.0,0.0,,-0.001,0.493,0.0,0.0,71,1065
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,963.0,891.0,1024.0,12.16,20.3,45.3,-14.7,,0.588,0.847,0.2,0.0,71,1065
|
data/subsets/tasks_all/models_all/budgets_1/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+45 / −44</span></td><td class='num' data-sort='6.31'>6.31</td><td class='num' data-sort='17.8'>17.8</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1346.0'>1346<span class='sub'>+42 / −43</span></td><td class='num' data-sort='6.51'>6.51</td><td class='num' data-sort='20.6'>20.6</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1335.0'>1335<span class='sub'>+40 / −39</span></td><td class='num' data-sort='6.63'>6.63</td><td class='num' data-sort='20.9'>20.9</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1323.0'>1323<span class='sub'>+44 / −43</span></td><td class='num' data-sort='6.78'>6.78</td><td class='num' data-sort='20.2'>20.2</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.3'>2.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1316.0'>1316<span class='sub'>+34 / −30</span></td><td class='num' data-sort='7.01'>7.01</td><td class='num' data-sort='21.2'>21.2</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1307.0'>1307<span class='sub'>+36 / −33</span></td><td class='num' data-sort='7.15'>7.15</td><td class='num' data-sort='22.8'>22.8</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1298.0'>1298<span class='sub'>+36 / −40</span></td><td class='num' data-sort='7.39'>7.39</td><td class='num' data-sort='23.2'>23.2</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+34 / −36</span></td><td class='num' data-sort='7.82'>7.82</td><td class='num' data-sort='25.1'>25.1</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1268.0'>1268<span class='sub'>+39 / −40</span></td><td class='num' data-sort='7.89'>7.89</td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1257.0'>1257<span class='sub'>+40 / −39</span></td><td class='num' data-sort='8.18'>8.18</td><td class='num' data-sort='24.4'>24.4</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1256.0'>1256<span class='sub'>+43 / −44</span></td><td class='num' data-sort='7.91'>7.91</td><td class='num' data-sort='25.0'>25.0</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1634.9'>1,634.9</td><td class='num' data-sort='43.8'>43.8</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1227.0'>1227<span class='sub'>+49 / −50</span></td><td class='num' data-sort='8.37'>8.37</td><td class='num' data-sort='28.2'>28.2</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3166.2'>3,166.2</td><td class='num' data-sort='48.9'>48.9</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1219.0'>1219<span class='sub'>+43 / −42</span></td><td class='num' data-sort='8.54'>8.54</td><td class='num' data-sort='27.9'>27.9</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='237.3'>237.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1025.0'>1025<span class='sub'>+52 / −64</span></td><td class='num' data-sort='11.85'>11.85</td><td class='num' data-sort='42.2'>42.2</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+41 / −47</span></td><td class='num' data-sort='11.64'>11.64</td><td class='num' data-sort='39.7'>39.7</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1366.0,1322.0,1411.0,
|
| 37 |
-
2,ElasticNet,Non-search,Regularization,1346.0,1303.0,1388.0,
|
| 38 |
-
3,CART,Non-search,Tree,1335.0,1296.0,1375.0,
|
| 39 |
-
4,Lasso,Non-search,Regularization,1323.0,1280.0,1367.0,
|
| 40 |
-
5,MI,Non-search,Information theory,1316.0,1286.0,1350.0,
|
| 41 |
-
6,MarkovBlanket,Search,Backward search,1307.0,1274.0,1343.0,
|
| 42 |
-
7,F-Test,Non-search,Correlation,1298.0,1258.0,1334.0,
|
| 43 |
-
8,GainRatio,Non-search,Information theory,1273.0,1237.0,1307.0,
|
| 44 |
-
9,mRMR,Non-search,Information theory,1268.0,1228.0,1307.0,
|
| 45 |
-
10,(R)ReliefF,Non-search,Distance,1257.0,1218.0,1297.0,
|
| 46 |
-
11,SFS,Search,Forward search,1256.0,1212.0,1299.0,
|
| 47 |
-
12,SBE,Search,Backward search,1227.0,1177.0,1276.0,
|
| 48 |
-
13,LOCO,Search,Leave-one-out,1219.0,1177.0,1262.0,
|
| 49 |
-
14,LaplacianScore,Non-search,Distance,1025.0,961.0,1077.0,
|
| 50 |
-
15,Random,Baseline,Baseline,1000.0,953.0,1041.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+45 / −44</span></td><td class='num' data-sort='29.3'>29.3</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1346.0'>1346<span class='sub'>+42 / −43</span></td><td class='num' data-sort='26.0'>26.0</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1335.0'>1335<span class='sub'>+40 / −39</span></td><td class='num' data-sort='27.1'>27.1</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1323.0'>1323<span class='sub'>+44 / −43</span></td><td class='num' data-sort='25.4'>25.4</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.3'>2.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1316.0'>1316<span class='sub'>+34 / −30</span></td><td class='num' data-sort='27.5'>27.5</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1307.0'>1307<span class='sub'>+36 / −33</span></td><td class='num' data-sort='24.6'>24.6</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1298.0'>1298<span class='sub'>+36 / −40</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+34 / −36</span></td><td class='num' data-sort='23.7'>23.7</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1268.0'>1268<span class='sub'>+39 / −40</span></td><td class='num' data-sort='24.6'>24.6</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1257.0'>1257<span class='sub'>+40 / −39</span></td><td class='num' data-sort='21.3'>21.3</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1256.0'>1256<span class='sub'>+43 / −44</span></td><td class='num' data-sort='21.6'>21.6</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1634.9'>1,634.9</td><td class='num' data-sort='43.8'>43.8</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1227.0'>1227<span class='sub'>+49 / −50</span></td><td class='num' data-sort='14.7'>14.7</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3166.2'>3,166.2</td><td class='num' data-sort='48.9'>48.9</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1219.0'>1219<span class='sub'>+43 / −42</span></td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='237.3'>237.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1025.0'>1025<span class='sub'>+52 / −64</span></td><td class='num' data-sort='-22.8'>-22.8</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+41 / −47</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1366.0,1322.0,1411.0,29.3,0.653,0.741,0.2,0.0
|
| 37 |
+
2,ElasticNet,Non-search,Regularization,1346.0,1303.0,1388.0,26.0,0.519,0.873,7.6,0.0
|
| 38 |
+
3,CART,Non-search,Tree,1335.0,1296.0,1375.0,27.1,0.456,0.638,0.1,0.0
|
| 39 |
+
4,Lasso,Non-search,Regularization,1323.0,1280.0,1367.0,25.4,0.422,0.825,2.3,0.0
|
| 40 |
+
5,MI,Non-search,Information theory,1316.0,1286.0,1350.0,27.5,0.673,0.863,0.4,0.0
|
| 41 |
+
6,MarkovBlanket,Search,Backward search,1307.0,1274.0,1343.0,24.6,0.396,0.467,0.2,0.0
|
| 42 |
+
7,F-Test,Non-search,Correlation,1298.0,1258.0,1334.0,25.9,0.734,0.932,0.0,0.0
|
| 43 |
+
8,GainRatio,Non-search,Information theory,1273.0,1237.0,1307.0,23.7,0.681,0.915,0.2,0.0
|
| 44 |
+
9,mRMR,Non-search,Information theory,1268.0,1228.0,1307.0,24.6,0.578,0.712,0.4,0.0
|
| 45 |
+
10,(R)ReliefF,Non-search,Distance,1257.0,1218.0,1297.0,21.3,0.647,0.808,14.3,0.0
|
| 46 |
+
11,SFS,Search,Forward search,1256.0,1212.0,1299.0,21.6,0.278,0.664,1634.9,43.8
|
| 47 |
+
12,SBE,Search,Backward search,1227.0,1177.0,1276.0,14.7,0.175,0.578,3166.2,48.9
|
| 48 |
+
13,LOCO,Search,Leave-one-out,1219.0,1177.0,1262.0,15.9,0.22,0.613,237.3,0.0
|
| 49 |
+
14,LaplacianScore,Non-search,Distance,1025.0,961.0,1077.0,-22.8,0.644,0.85,0.2,0.0
|
| 50 |
+
15,Random,Baseline,Baseline,1000.0,953.0,1041.0,0.0,0.001,0.499,0.0,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_all/budgets_1/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1366.0,1322.0,1411.0,6.31,62.1,17.8,,0.653,0.741,0.2,0.0,71,1065
|
| 3 |
-
2,ElasticNet,Non-search,Regularization,1346.0,1303.0,1388.0,6.51,60.6,20.6,,0.519,0.873,7.6,0.0,71,1065
|
| 4 |
-
3,CART,Non-search,Tree,1335.0,1296.0,1375.0,6.63,59.8,20.9,,0.456,0.638,0.1,0.0,71,1065
|
| 5 |
-
4,Lasso,Non-search,Regularization,1323.0,1280.0,1367.0,6.78,58.7,20.2,,0.422,0.825,2.3,0.0,71,1065
|
| 6 |
-
5,MI,Non-search,Information theory,1316.0,1286.0,1350.0,7.01,57.1,21.2,,0.673,0.863,0.4,0.0,71,1065
|
| 7 |
-
6,MarkovBlanket,Search,Backward search,1307.0,1274.0,1343.0,7.15,56.0,22.8,,0.396,0.467,0.2,0.0,71,1065
|
| 8 |
-
7,F-Test,Non-search,Correlation,1298.0,1258.0,1334.0,7.39,54.3,23.2,,0.734,0.932,0.0,0.0,71,1065
|
| 9 |
-
8,GainRatio,Non-search,Information theory,1273.0,1237.0,1307.0,7.82,51.3,25.1,,0.681,0.915,0.2,0.0,71,1065
|
| 10 |
-
9,mRMR,Non-search,Information theory,1268.0,1228.0,1307.0,7.89,50.8,23.8,,0.578,0.712,0.4,0.0,71,1065
|
| 11 |
-
10,(R)ReliefF,Non-search,Distance,1257.0,1218.0,1297.0,8.18,48.7,24.4,,0.647,0.808,14.3,0.0,71,1065
|
| 12 |
-
11,SFS,Search,Forward search,1256.0,1212.0,1299.0,7.91,50.6,25.0,,0.278,0.664,1634.9,43.8,71,1065
|
| 13 |
-
12,SBE,Search,Backward search,1227.0,1177.0,1276.0,8.37,47.3,28.2,,0.175,0.578,3166.2,48.9,71,1065
|
| 14 |
-
13,LOCO,Search,Leave-one-out,1219.0,1177.0,1262.0,8.54,46.1,27.9,,0.22,0.613,237.3,0.0,71,1065
|
| 15 |
-
14,LaplacianScore,Non-search,Distance,1025.0,961.0,1077.0,11.85,22.5,42.2,,0.644,0.85,0.2,0.0,71,1065
|
| 16 |
-
15,Random,Baseline,Baseline,1000.0,953.0,1041.0,11.64,24.0,39.7,,0.001,0.499,0.0,0.0,71,1065
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1366.0,1322.0,1411.0,6.31,62.1,17.8,29.3,,0.653,0.741,0.2,0.0,71,1065
|
| 3 |
+
2,ElasticNet,Non-search,Regularization,1346.0,1303.0,1388.0,6.51,60.6,20.6,26.0,,0.519,0.873,7.6,0.0,71,1065
|
| 4 |
+
3,CART,Non-search,Tree,1335.0,1296.0,1375.0,6.63,59.8,20.9,27.1,,0.456,0.638,0.1,0.0,71,1065
|
| 5 |
+
4,Lasso,Non-search,Regularization,1323.0,1280.0,1367.0,6.78,58.7,20.2,25.4,,0.422,0.825,2.3,0.0,71,1065
|
| 6 |
+
5,MI,Non-search,Information theory,1316.0,1286.0,1350.0,7.01,57.1,21.2,27.5,,0.673,0.863,0.4,0.0,71,1065
|
| 7 |
+
6,MarkovBlanket,Search,Backward search,1307.0,1274.0,1343.0,7.15,56.0,22.8,24.6,,0.396,0.467,0.2,0.0,71,1065
|
| 8 |
+
7,F-Test,Non-search,Correlation,1298.0,1258.0,1334.0,7.39,54.3,23.2,25.9,,0.734,0.932,0.0,0.0,71,1065
|
| 9 |
+
8,GainRatio,Non-search,Information theory,1273.0,1237.0,1307.0,7.82,51.3,25.1,23.7,,0.681,0.915,0.2,0.0,71,1065
|
| 10 |
+
9,mRMR,Non-search,Information theory,1268.0,1228.0,1307.0,7.89,50.8,23.8,24.6,,0.578,0.712,0.4,0.0,71,1065
|
| 11 |
+
10,(R)ReliefF,Non-search,Distance,1257.0,1218.0,1297.0,8.18,48.7,24.4,21.3,,0.647,0.808,14.3,0.0,71,1065
|
| 12 |
+
11,SFS,Search,Forward search,1256.0,1212.0,1299.0,7.91,50.6,25.0,21.6,,0.278,0.664,1634.9,43.8,71,1065
|
| 13 |
+
12,SBE,Search,Backward search,1227.0,1177.0,1276.0,8.37,47.3,28.2,14.7,,0.175,0.578,3166.2,48.9,71,1065
|
| 14 |
+
13,LOCO,Search,Leave-one-out,1219.0,1177.0,1262.0,8.54,46.1,27.9,15.9,,0.22,0.613,237.3,0.0,71,1065
|
| 15 |
+
14,LaplacianScore,Non-search,Distance,1025.0,961.0,1077.0,11.85,22.5,42.2,-22.8,,0.644,0.85,0.2,0.0,71,1065
|
| 16 |
+
15,Random,Baseline,Baseline,1000.0,953.0,1041.0,11.64,24.0,39.7,0.0,,0.001,0.499,0.0,0.0,71,1065
|
data/subsets/tasks_all/models_all/budgets_2/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+45 / −39</span></td><td class='num' data-sort='6.04'>6.04</td><td class='num' data-sort='14.6'>14.6</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1278.0'>1278<span class='sub'>+42 / −37</span></td><td class='num' data-sort='6.67'>6.67</td><td class='num' data-sort='18.5'>18.5</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+42 / −43</span></td><td class='num' data-sort='6.64'>6.64</td><td class='num' data-sort='18.1'>18.1</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1272.0'>1272<span class='sub'>+40 / −37</span></td><td class='num' data-sort='6.82'>6.82</td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1249.0'>1249<span class='sub'>+40 / −45</span></td><td class='num' data-sort='6.97'>6.97</td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1237.0'>1237<span class='sub'>+32 / −34</span></td><td class='num' data-sort='7.5'>7.50</td><td class='num' data-sort='21.0'>21.0</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1223.0'>1223<span class='sub'>+36 / −35</span></td><td class='num' data-sort='7.49'>7.49</td><td class='num' data-sort='20.5'>20.5</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1210.0'>1210<span class='sub'>+39 / −38</span></td><td class='num' data-sort='7.86'>7.86</td><td class='num' data-sort='20.2'>20.2</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='14.4'>14.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1194.0'>1194<span class='sub'>+38 / −37</span></td><td class='num' data-sort='8.22'>8.22</td><td class='num' data-sort='22.8'>22.8</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+45 / −46</span></td><td class='num' data-sort='8.15'>8.15</td><td class='num' data-sort='24.7'>24.7</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2621.7'>2,621.7</td><td class='num' data-sort='47.8'>47.8</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1181.0'>1181<span class='sub'>+35 / −37</span></td><td class='num' data-sort='8.22'>8.22</td><td class='num' data-sort='21.5'>21.5</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1175.0'>1175<span class='sub'>+37 / −42</span></td><td class='num' data-sort='8.11'>8.11</td><td class='num' data-sort='23.1'>23.1</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2398.9'>2,398.9</td><td class='num' data-sort='47.2'>47.2</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1157.0'>1157<span class='sub'>+40 / −41</span></td><td class='num' data-sort='8.42'>8.42</td><td class='num' data-sort='25.1'>25.1</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='241.3'>241.3</td><td class='num' data-sort='6.8'>6.8</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+42 / −49</span></td><td class='num' data-sort='10.82'>10.82</td><td class='num' data-sort='33.5'>33.5</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='930.0'>930<span class='sub'>+54 / −71</span></td><td class='num' data-sort='12.08'>12.08</td><td class='num' data-sort='39.5'>39.5</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1325.0,1286.0,1370.0,
|
| 37 |
-
2,CART,Non-search,Tree,1278.0,1241.0,1320.0,
|
| 38 |
-
3,ElasticNet,Non-search,Regularization,1273.0,1230.0,1315.0,
|
| 39 |
-
4,MI,Non-search,Information theory,1272.0,1235.0,1312.0,
|
| 40 |
-
5,Lasso,Non-search,Regularization,1249.0,1204.0,1289.0,
|
| 41 |
-
6,MarkovBlanket,Search,Backward search,1237.0,1203.0,1269.0,
|
| 42 |
-
7,F-Test,Non-search,Correlation,1223.0,1188.0,1259.0,
|
| 43 |
-
8,(R)ReliefF,Non-search,Distance,1210.0,1172.0,1249.0,
|
| 44 |
-
9,GainRatio,Non-search,Information theory,1194.0,1157.0,1232.0,
|
| 45 |
-
10,SBE,Search,Backward search,1186.0,1140.0,1231.0,
|
| 46 |
-
11,mRMR,Non-search,Information theory,1181.0,1144.0,1216.0,
|
| 47 |
-
12,SFS,Search,Forward search,1175.0,1133.0,1212.0,
|
| 48 |
-
13,LOCO,Search,Leave-one-out,1157.0,1116.0,1197.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,951.0,1042.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,930.0,859.0,984.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+45 / −39</span></td><td class='num' data-sort='24.3'>24.3</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1278.0'>1278<span class='sub'>+42 / −37</span></td><td class='num' data-sort='20.5'>20.5</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+42 / −43</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1272.0'>1272<span class='sub'>+40 / −37</span></td><td class='num' data-sort='20.9'>20.9</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1249.0'>1249<span class='sub'>+40 / −45</span></td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1237.0'>1237<span class='sub'>+32 / −34</span></td><td class='num' data-sort='18.0'>18.0</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1223.0'>1223<span class='sub'>+36 / −35</span></td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1210.0'>1210<span class='sub'>+39 / −38</span></td><td class='num' data-sort='18.5'>18.5</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='14.4'>14.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1194.0'>1194<span class='sub'>+38 / −37</span></td><td class='num' data-sort='16.2'>16.2</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+45 / −46</span></td><td class='num' data-sort='10.4'>10.4</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2621.7'>2,621.7</td><td class='num' data-sort='47.8'>47.8</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1181.0'>1181<span class='sub'>+35 / −37</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1175.0'>1175<span class='sub'>+37 / −42</span></td><td class='num' data-sort='13.6'>13.6</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2398.9'>2,398.9</td><td class='num' data-sort='47.2'>47.2</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1157.0'>1157<span class='sub'>+40 / −41</span></td><td class='num' data-sort='9.3'>9.3</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='241.3'>241.3</td><td class='num' data-sort='6.8'>6.8</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+42 / −49</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='930.0'>930<span class='sub'>+54 / −71</span></td><td class='num' data-sort='-58.2'>-58.2</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1325.0,1286.0,1370.0,24.3,0.676,0.691,0.2,0.0
|
| 37 |
+
2,CART,Non-search,Tree,1278.0,1241.0,1320.0,20.5,0.444,0.572,0.1,0.0
|
| 38 |
+
3,ElasticNet,Non-search,Regularization,1273.0,1230.0,1315.0,19.8,0.481,0.777,7.6,0.0
|
| 39 |
+
4,MI,Non-search,Information theory,1272.0,1235.0,1312.0,20.9,0.683,0.815,0.4,0.0
|
| 40 |
+
5,Lasso,Non-search,Regularization,1249.0,1204.0,1289.0,18.3,0.389,0.73,2.0,0.0
|
| 41 |
+
6,MarkovBlanket,Search,Backward search,1237.0,1203.0,1269.0,18.0,0.413,0.447,0.1,0.0
|
| 42 |
+
7,F-Test,Non-search,Correlation,1223.0,1188.0,1259.0,18.3,0.706,0.879,0.0,0.0
|
| 43 |
+
8,(R)ReliefF,Non-search,Distance,1210.0,1172.0,1249.0,18.5,0.673,0.757,14.4,0.0
|
| 44 |
+
9,GainRatio,Non-search,Information theory,1194.0,1157.0,1232.0,16.2,0.67,0.856,0.2,0.0
|
| 45 |
+
10,SBE,Search,Backward search,1186.0,1140.0,1231.0,10.4,0.144,0.556,2621.7,47.8
|
| 46 |
+
11,mRMR,Non-search,Information theory,1181.0,1144.0,1216.0,17.7,0.614,0.66,0.5,0.0
|
| 47 |
+
12,SFS,Search,Forward search,1175.0,1133.0,1212.0,13.6,0.239,0.627,2398.9,47.2
|
| 48 |
+
13,LOCO,Search,Leave-one-out,1157.0,1116.0,1197.0,9.3,0.151,0.575,241.3,6.8
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,951.0,1042.0,0.0,-0.001,0.513,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,930.0,859.0,984.0,-58.2,0.66,0.827,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_all/budgets_2/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='135.7' x2='135.7' y1='16.2' y2='368.0'/><text class='sa-text' x='135.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='240.2' x2='240.2' y1='16.2' y2='368.0'/><text class='sa-text' x='240.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='344.6' x2='344.6' y1='16.2' y2='368.0'/><text class='sa-text' x='344.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='449.1' x2='449.1' y1='16.2' y2='368.0'/><text class='sa-text' x='449.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='553.5' x2='553.5' y1='16.2' y2='368.0'/><text class='sa-text' x='553.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='658.0' x2='658.0' y1='16.2' y2='368.0'/><text class='sa-text' x='658.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='762.4' x2='762.4' y1='16.2' y2='368.0'/><text class='sa-text' x='762.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='286.2' y2='286.2'/><text class='sa-text' x='57.9' y='292.2' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='220.4' y2='220.4'/><text class='sa-text' x='57.9' y='226.4' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='154.6' y2='154.6'/><text class='sa-text' x='57.9' y='160.6' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='88.8' y2='88.8'/><text class='sa-text' x='57.9' y='94.8' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='23.0' y2='23.0'/><text class='sa-text' x='57.9' y='29.0' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 333.9,111.7 469.8,82.6 517.4,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='517.4' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='523.2' cy='77.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='469.8' cy='82.6' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='697.8' cy='91.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='803.0' cy='108.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='697.7' cy='111.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='333.9' cy='111.7' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='483.8' cy='114.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='510.3' cy='118.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='465.2' cy='138.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='173.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='810.9' cy='215.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='741.9' cy='229.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='517.4' y1='32.2' x2='641.4' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='641.4' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-dim sa-halo' x='541.4' y='73.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><text class='sa-text sa-halo' x='477.3' y='63.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='697.8' y1='91.9' x2='688.3' y2='66.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='688.3' y='66.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='803.0' y1='108.3' x2='763.6' y2='98.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='763.6' y='98.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='697.7' y1='111.1' x2='720.3' y2='131.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='720.3' y='131.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='333.9' y1='111.7' x2='379.2' y2='131.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='379.2' y='131.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='483.8' y1='114.6' x2='377.4' y2='93.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='377.4' y='93.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='510.3' y1='118.8' x2='549.9' y2='104.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='549.9' y='104.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='465.2' y1='138.7' x2='536.3' y2='152.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='536.3' y='152.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='812.3' y1='173.5' x2='828.6' y2='159.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='159.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='810.9' y1='215.6' x2='827.3' y2='229.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='827.3' y='229.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='741.9' y1='229.4' x2='709.1' y2='243.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='709.1' y='243.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='517.4' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 24.3% Selection time per 1k features: 45.1 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 24.3% Selection time per 1k features: 45.1 s Datasets: 71'/><circle class='sa-hit' cx='523.2' cy='77.1' r='10.5' data-tip='MI Improvement over Random: 20.9% Selection time per 1k features: 51.2 s Datasets: 71' aria-label='MI Improvement over Random: 20.9% Selection time per 1k features: 51.2 s Datasets: 71'/><circle class='sa-hit' cx='469.8' cy='82.6' r='10.5' data-tip='CART (on the Pareto front) Improvement over Random: 20.5% Selection time per 1k features: 15.8 s Datasets: 71' aria-label='CART (on the Pareto front) Improvement over Random: 20.5% Selection time per 1k features: 15.8 s Datasets: 71'/><circle class='sa-hit' cx='697.8' cy='91.9' r='10.5' data-tip='ElasticNet Improvement over Random: 19.8% Selection time per 1k features: 2.4e+03 s Datasets: 71 Next to Lasso: 18.3%, 2.4e+03 s' aria-label='ElasticNet Improvement over Random: 19.8% Selection time per 1k features: 2.4e+03 s Datasets: 71 Next to Lasso: 18.3%, 2.4e+03 s'/><circle class='sa-hit' cx='803.0' cy='108.3' r='10.5' data-tip='(R)ReliefF Improvement over Random: 18.5% Selection time per 1k features: 2.44e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 18.5% Selection time per 1k features: 2.44e+04 s Datasets: 71'/><circle class='sa-hit' cx='697.7' cy='111.1' r='10.5' data-tip='Lasso Improvement over Random: 18.3% Selection time per 1k features: 2.4e+03 s Datasets: 71 Next to ElasticNet: 19.8%, 2.4e+03 s' aria-label='Lasso Improvement over Random: 18.3% Selection time per 1k features: 2.4e+03 s Datasets: 71 Next to ElasticNet: 19.8%, 2.4e+03 s'/><circle class='sa-hit' cx='333.9' cy='111.7' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 18.3% Selection time per 1k features: 0.789 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 18.3% Selection time per 1k features: 0.789 s Datasets: 71'/><circle class='sa-hit' cx='483.8' cy='114.6' r='10.5' data-tip='MarkovBlanket Improvement over Random: 18.0% Selection time per 1k features: 21.5 s Datasets: 71' aria-label='MarkovBlanket Improvement over Random: 18.0% Selection time per 1k features: 21.5 s Datasets: 71'/><circle class='sa-hit' cx='510.3' cy='118.8' r='10.5' data-tip='mRMR Improvement over Random: 17.7% Selection time per 1k features: 38.5 s Datasets: 71' aria-label='mRMR Improvement over Random: 17.7% Selection time per 1k features: 38.5 s Datasets: 71'/><circle class='sa-hit' cx='465.2' cy='138.7' r='10.5' data-tip='GainRatio Improvement over Random: 16.2% Selection time per 1k features: 14.3 s Datasets: 71' aria-label='GainRatio Improvement over Random: 16.2% Selection time per 1k features: 14.3 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='173.5' r='10.5' data-tip='SFS Improvement over Random: 13.6% Selection time per 1k features: 3e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 13.6% Selection time per 1k features: 3e+04 s Datasets: 71'/><circle class='sa-hit' cx='810.9' cy='215.6' r='10.5' data-tip='SBE Improvement over Random: 10.4% Selection time per 1k features: 2.91e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 10.4% Selection time per 1k features: 2.91e+04 s Datasets: 71'/><circle class='sa-hit' cx='741.9' cy='229.4' r='10.5' data-tip='LOCO Improvement over Random: 9.3% Selection time per 1k features: 6.35e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 9.3% Selection time per 1k features: 6.35e+03 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0048 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0048 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,24.3008,45.0766,71,True
|
| 37 |
MI,20.8893,51.2055,71,False
|
| 38 |
CART,20.4694,15.7754,71,True
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='135.7' x2='135.7' y1='16.2' y2='368.0'/><text class='sa-text' x='135.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='240.2' x2='240.2' y1='16.2' y2='368.0'/><text class='sa-text' x='240.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='344.6' x2='344.6' y1='16.2' y2='368.0'/><text class='sa-text' x='344.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='449.1' x2='449.1' y1='16.2' y2='368.0'/><text class='sa-text' x='449.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='553.5' x2='553.5' y1='16.2' y2='368.0'/><text class='sa-text' x='553.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='658.0' x2='658.0' y1='16.2' y2='368.0'/><text class='sa-text' x='658.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='762.4' x2='762.4' y1='16.2' y2='368.0'/><text class='sa-text' x='762.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='286.2' y2='286.2'/><text class='sa-text' x='57.9' y='292.2' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='220.4' y2='220.4'/><text class='sa-text' x='57.9' y='226.4' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='154.6' y2='154.6'/><text class='sa-text' x='57.9' y='160.6' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='88.8' y2='88.8'/><text class='sa-text' x='57.9' y='94.8' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='23.0' y2='23.0'/><text class='sa-text' x='57.9' y='29.0' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 333.9,111.7 469.8,82.6 517.4,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='517.4' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='523.2' cy='77.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='469.8' cy='82.6' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='697.8' cy='91.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='803.0' cy='108.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='697.7' cy='111.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='333.9' cy='111.7' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='483.8' cy='114.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='510.3' cy='118.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='465.2' cy='138.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='173.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='810.9' cy='215.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='741.9' cy='229.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='517.4' y1='32.2' x2='641.4' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='641.4' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-dim sa-halo' x='541.4' y='73.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><text class='sa-text sa-halo' x='477.3' y='63.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='697.8' y1='91.9' x2='692.1' y2='70.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='692.1' y='70.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='803.0' y1='108.3' x2='758.5' y2='98.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='758.5' y='98.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='697.7' y1='111.1' x2='721.6' y2='128.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='721.6' y='128.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='333.9' y1='111.7' x2='378.6' y2='130.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='378.6' y='130.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='483.8' y1='114.6' x2='374.9' y2='94.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='374.9' y='94.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='510.3' y1='118.8' x2='549.9' y2='104.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='549.9' y='104.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='465.2' y1='138.7' x2='536.3' y2='152.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='536.3' y='152.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='812.3' y1='173.5' x2='828.6' y2='159.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='159.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='810.9' y1='215.6' x2='827.3' y2='229.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='827.3' y='229.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='741.9' y1='229.4' x2='709.1' y2='243.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='709.1' y='243.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='517.4' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 24.3% Selection time per 1k features: 45.1 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 24.3% Selection time per 1k features: 45.1 s Datasets: 71'/><circle class='sa-hit' cx='523.2' cy='77.1' r='10.5' data-tip='MI Improvement over Random: 20.9% Selection time per 1k features: 51.2 s Datasets: 71' aria-label='MI Improvement over Random: 20.9% Selection time per 1k features: 51.2 s Datasets: 71'/><circle class='sa-hit' cx='469.8' cy='82.6' r='10.5' data-tip='CART (on the Pareto front) Improvement over Random: 20.5% Selection time per 1k features: 15.8 s Datasets: 71' aria-label='CART (on the Pareto front) Improvement over Random: 20.5% Selection time per 1k features: 15.8 s Datasets: 71'/><circle class='sa-hit' cx='697.8' cy='91.9' r='10.5' data-tip='ElasticNet Improvement over Random: 19.8% Selection time per 1k features: 2.4e+03 s Datasets: 71 Next to Lasso: 18.3%, 2.4e+03 s' aria-label='ElasticNet Improvement over Random: 19.8% Selection time per 1k features: 2.4e+03 s Datasets: 71 Next to Lasso: 18.3%, 2.4e+03 s'/><circle class='sa-hit' cx='803.0' cy='108.3' r='10.5' data-tip='(R)ReliefF Improvement over Random: 18.5% Selection time per 1k features: 2.44e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 18.5% Selection time per 1k features: 2.44e+04 s Datasets: 71'/><circle class='sa-hit' cx='697.7' cy='111.1' r='10.5' data-tip='Lasso Improvement over Random: 18.3% Selection time per 1k features: 2.4e+03 s Datasets: 71 Next to ElasticNet: 19.8%, 2.4e+03 s' aria-label='Lasso Improvement over Random: 18.3% Selection time per 1k features: 2.4e+03 s Datasets: 71 Next to ElasticNet: 19.8%, 2.4e+03 s'/><circle class='sa-hit' cx='333.9' cy='111.7' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 18.3% Selection time per 1k features: 0.789 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 18.3% Selection time per 1k features: 0.789 s Datasets: 71'/><circle class='sa-hit' cx='483.8' cy='114.6' r='10.5' data-tip='MarkovBlanket Improvement over Random: 18.0% Selection time per 1k features: 21.5 s Datasets: 71' aria-label='MarkovBlanket Improvement over Random: 18.0% Selection time per 1k features: 21.5 s Datasets: 71'/><circle class='sa-hit' cx='510.3' cy='118.8' r='10.5' data-tip='mRMR Improvement over Random: 17.7% Selection time per 1k features: 38.5 s Datasets: 71' aria-label='mRMR Improvement over Random: 17.7% Selection time per 1k features: 38.5 s Datasets: 71'/><circle class='sa-hit' cx='465.2' cy='138.7' r='10.5' data-tip='GainRatio Improvement over Random: 16.2% Selection time per 1k features: 14.3 s Datasets: 71' aria-label='GainRatio Improvement over Random: 16.2% Selection time per 1k features: 14.3 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='173.5' r='10.5' data-tip='SFS Improvement over Random: 13.6% Selection time per 1k features: 3e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 13.6% Selection time per 1k features: 3e+04 s Datasets: 71'/><circle class='sa-hit' cx='810.9' cy='215.6' r='10.5' data-tip='SBE Improvement over Random: 10.4% Selection time per 1k features: 2.91e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 10.4% Selection time per 1k features: 2.91e+04 s Datasets: 71'/><circle class='sa-hit' cx='741.9' cy='229.4' r='10.5' data-tip='LOCO Improvement over Random: 9.3% Selection time per 1k features: 6.35e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 9.3% Selection time per 1k features: 6.35e+03 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0048 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0048 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,24.3008,45.0766,71,True
|
| 37 |
MI,20.8893,51.2055,71,False
|
| 38 |
CART,20.4694,15.7754,71,True
|
data/subsets/tasks_all/models_all/budgets_2/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1325.0,1286.0,1370.0,6.04,64.0,14.6,,0.676,0.691,0.2,0.0,71,1065
|
| 3 |
-
2,CART,Non-search,Tree,1278.0,1241.0,1320.0,6.67,59.5,18.5,,0.444,0.572,0.1,0.0,71,1065
|
| 4 |
-
3,ElasticNet,Non-search,Regularization,1273.0,1230.0,1315.0,6.64,59.7,18.1,,0.481,0.777,7.6,0.0,71,1065
|
| 5 |
-
4,MI,Non-search,Information theory,1272.0,1235.0,1312.0,6.82,58.4,18.3,,0.683,0.815,0.4,0.0,71,1065
|
| 6 |
-
5,Lasso,Non-search,Regularization,1249.0,1204.0,1289.0,6.97,57.4,18.3,,0.389,0.73,2.0,0.0,71,1065
|
| 7 |
-
6,MarkovBlanket,Search,Backward search,1237.0,1203.0,1269.0,7.5,53.6,21.0,,0.413,0.447,0.1,0.0,71,1065
|
| 8 |
-
7,F-Test,Non-search,Correlation,1223.0,1188.0,1259.0,7.49,53.6,20.5,,0.706,0.879,0.0,0.0,71,1065
|
| 9 |
-
8,(R)ReliefF,Non-search,Distance,1210.0,1172.0,1249.0,7.86,51.0,20.2,,0.673,0.757,14.4,0.0,71,1065
|
| 10 |
-
9,GainRatio,Non-search,Information theory,1194.0,1157.0,1232.0,8.22,48.4,22.8,,0.67,0.856,0.2,0.0,71,1065
|
| 11 |
-
10,SBE,Search,Backward search,1186.0,1140.0,1231.0,8.15,48.9,24.7,,0.144,0.556,2621.7,47.8,71,1065
|
| 12 |
-
11,mRMR,Non-search,Information theory,1181.0,1144.0,1216.0,8.22,48.4,21.5,,0.614,0.66,0.5,0.0,71,1065
|
| 13 |
-
12,SFS,Search,Forward search,1175.0,1133.0,1212.0,8.11,49.2,23.1,,0.239,0.627,2398.9,47.2,71,1065
|
| 14 |
-
13,LOCO,Search,Leave-one-out,1157.0,1116.0,1197.0,8.42,47.0,25.1,,0.151,0.575,241.3,6.8,71,1065
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,951.0,1042.0,10.82,29.9,33.5,,-0.001,0.513,0.0,0.0,71,1065
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,930.0,859.0,984.0,12.08,20.9,39.5,,0.66,0.827,0.2,0.0,71,1065
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1325.0,1286.0,1370.0,6.04,64.0,14.6,24.3,,0.676,0.691,0.2,0.0,71,1065
|
| 3 |
+
2,CART,Non-search,Tree,1278.0,1241.0,1320.0,6.67,59.5,18.5,20.5,,0.444,0.572,0.1,0.0,71,1065
|
| 4 |
+
3,ElasticNet,Non-search,Regularization,1273.0,1230.0,1315.0,6.64,59.7,18.1,19.8,,0.481,0.777,7.6,0.0,71,1065
|
| 5 |
+
4,MI,Non-search,Information theory,1272.0,1235.0,1312.0,6.82,58.4,18.3,20.9,,0.683,0.815,0.4,0.0,71,1065
|
| 6 |
+
5,Lasso,Non-search,Regularization,1249.0,1204.0,1289.0,6.97,57.4,18.3,18.3,,0.389,0.73,2.0,0.0,71,1065
|
| 7 |
+
6,MarkovBlanket,Search,Backward search,1237.0,1203.0,1269.0,7.5,53.6,21.0,18.0,,0.413,0.447,0.1,0.0,71,1065
|
| 8 |
+
7,F-Test,Non-search,Correlation,1223.0,1188.0,1259.0,7.49,53.6,20.5,18.3,,0.706,0.879,0.0,0.0,71,1065
|
| 9 |
+
8,(R)ReliefF,Non-search,Distance,1210.0,1172.0,1249.0,7.86,51.0,20.2,18.5,,0.673,0.757,14.4,0.0,71,1065
|
| 10 |
+
9,GainRatio,Non-search,Information theory,1194.0,1157.0,1232.0,8.22,48.4,22.8,16.2,,0.67,0.856,0.2,0.0,71,1065
|
| 11 |
+
10,SBE,Search,Backward search,1186.0,1140.0,1231.0,8.15,48.9,24.7,10.4,,0.144,0.556,2621.7,47.8,71,1065
|
| 12 |
+
11,mRMR,Non-search,Information theory,1181.0,1144.0,1216.0,8.22,48.4,21.5,17.7,,0.614,0.66,0.5,0.0,71,1065
|
| 13 |
+
12,SFS,Search,Forward search,1175.0,1133.0,1212.0,8.11,49.2,23.1,13.6,,0.239,0.627,2398.9,47.2,71,1065
|
| 14 |
+
13,LOCO,Search,Leave-one-out,1157.0,1116.0,1197.0,8.42,47.0,25.1,9.3,,0.151,0.575,241.3,6.8,71,1065
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,951.0,1042.0,10.82,29.9,33.5,0.0,,-0.001,0.513,0.0,0.0,71,1065
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,930.0,859.0,984.0,12.08,20.9,39.5,-58.2,,0.66,0.827,0.2,0.0,71,1065
|
data/subsets/tasks_all/models_all/budgets_3/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1288.0'>1288<span class='sub'>+43 / −40</span></td><td class='num' data-sort='5.9'>5.90</td><td class='num' data-sort='12.4'>12.4</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1232.0'>1232<span class='sub'>+40 / −31</span></td><td class='num' data-sort='6.65'>6.65</td><td class='num' data-sort='15.6'>15.6</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1218.0'>1218<span class='sub'>+38 / −35</span></td><td class='num' data-sort='6.94'>6.94</td><td class='num' data-sort='15.8'>15.8</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1212.0'>1212<span class='sub'>+39 / −41</span></td><td class='num' data-sort='7.02'>7.02</td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+38 / −38</span></td><td class='num' data-sort='7.27'>7.27</td><td class='num' data-sort='16.4'>16.4</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+33 / −32</span></td><td class='num' data-sort='7.37'>7.37</td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+38 / −35</span></td><td class='num' data-sort='7.8'>7.80</td><td class='num' data-sort='17.0'>17.0</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1158.0'>1158<span class='sub'>+31 / −36</span></td><td class='num' data-sort='8.15'>8.15</td><td class='num' data-sort='18.6'>18.6</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1155.0'>1155<span class='sub'>+44 / −49</span></td><td class='num' data-sort='8.03'>8.03</td><td class='num' data-sort='19.9'>19.9</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1890.2'>1,890.2</td><td class='num' data-sort='44.9'>44.9</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1155.0'>1155<span class='sub'>+35 / −36</span></td><td class='num' data-sort='8.09'>8.09</td><td class='num' data-sort='17.2'>17.2</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1134.0'>1134<span class='sub'>+34 / −34</span></td><td class='num' data-sort='8.44'>8.44</td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1127.0'>1127<span class='sub'>+38 / −36</span></td><td class='num' data-sort='8.21'>8.21</td><td class='num' data-sort='20.4'>20.4</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='246.0'>246.0</td><td class='num' data-sort='12.3'>12.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1124.0'>1124<span class='sub'>+33 / −36</span></td><td class='num' data-sort='8.32'>8.32</td><td class='num' data-sort='20.2'>20.2</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2771.3'>2,771.3</td><td class='num' data-sort='48.7'>48.7</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+40 / −46</span></td><td class='num' data-sort='10.09'>10.09</td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='929.0'>929<span class='sub'>+52 / −60</span></td><td class='num' data-sort='11.73'>11.73</td><td class='num' data-sort='33.1'>33.1</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1288.0,1248.0,1331.0,
|
| 37 |
-
2,CART,Non-search,Tree,1232.0,1201.0,1272.0,
|
| 38 |
-
3,MI,Non-search,Information theory,1218.0,1183.0,1256.0,
|
| 39 |
-
4,ElasticNet,Non-search,Regularization,1212.0,1171.0,1251.0,
|
| 40 |
-
5,Lasso,Non-search,Regularization,1192.0,1154.0,1230.0,
|
| 41 |
-
6,F-Test,Non-search,Correlation,1189.0,1157.0,1222.0,
|
| 42 |
-
7,(R)ReliefF,Non-search,Distance,1172.0,1137.0,1210.0,
|
| 43 |
-
8,MarkovBlanket,Search,Backward search,1158.0,1122.0,1189.0,
|
| 44 |
-
9,SBE,Search,Backward search,1155.0,1106.0,1199.0,
|
| 45 |
-
10,mRMR,Non-search,Information theory,1155.0,1119.0,1190.0,
|
| 46 |
-
11,GainRatio,Non-search,Information theory,1134.0,1100.0,1168.0,
|
| 47 |
-
12,LOCO,Search,Leave-one-out,1127.0,1091.0,1165.0,
|
| 48 |
-
13,SFS,Search,Forward search,1124.0,1088.0,1157.0,8.
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,954.0,1040.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,929.0,869.0,981.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1288.0'>1288<span class='sub'>+43 / −40</span></td><td class='num' data-sort='18.0'>18.0</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1232.0'>1232<span class='sub'>+40 / −31</span></td><td class='num' data-sort='14.4'>14.4</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1218.0'>1218<span class='sub'>+38 / −35</span></td><td class='num' data-sort='14.2'>14.2</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1212.0'>1212<span class='sub'>+39 / −41</span></td><td class='num' data-sort='13.5'>13.5</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+38 / −38</span></td><td class='num' data-sort='12.6'>12.6</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+33 / −32</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+38 / −35</span></td><td class='num' data-sort='12.9'>12.9</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1158.0'>1158<span class='sub'>+31 / −36</span></td><td class='num' data-sort='11.3'>11.3</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1155.0'>1155<span class='sub'>+44 / −49</span></td><td class='num' data-sort='6.3'>6.3</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1890.2'>1,890.2</td><td class='num' data-sort='44.9'>44.9</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1155.0'>1155<span class='sub'>+35 / −36</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1134.0'>1134<span class='sub'>+34 / −34</span></td><td class='num' data-sort='11.6'>11.6</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1127.0'>1127<span class='sub'>+38 / −36</span></td><td class='num' data-sort='5.0'>5.0</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='246.0'>246.0</td><td class='num' data-sort='12.3'>12.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1124.0'>1124<span class='sub'>+33 / −36</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2771.3'>2,771.3</td><td class='num' data-sort='48.7'>48.7</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+40 / −46</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='929.0'>929<span class='sub'>+52 / −60</span></td><td class='num' data-sort='-89.9'>-89.9</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1288.0,1248.0,1331.0,18.0,0.661,0.6,0.2,0.0
|
| 37 |
+
2,CART,Non-search,Tree,1232.0,1201.0,1272.0,14.4,0.402,0.507,0.1,0.0
|
| 38 |
+
3,MI,Non-search,Information theory,1218.0,1183.0,1256.0,14.2,0.649,0.723,0.4,0.0
|
| 39 |
+
4,ElasticNet,Non-search,Regularization,1212.0,1171.0,1251.0,13.5,0.419,0.665,7.6,0.0
|
| 40 |
+
5,Lasso,Non-search,Regularization,1192.0,1154.0,1230.0,12.6,0.334,0.643,2.0,0.0
|
| 41 |
+
6,F-Test,Non-search,Correlation,1189.0,1157.0,1222.0,13.7,0.668,0.781,0.0,0.0
|
| 42 |
+
7,(R)ReliefF,Non-search,Distance,1172.0,1137.0,1210.0,12.9,0.656,0.667,14.3,0.0
|
| 43 |
+
8,MarkovBlanket,Search,Backward search,1158.0,1122.0,1189.0,11.3,0.434,0.473,0.1,0.0
|
| 44 |
+
9,SBE,Search,Backward search,1155.0,1106.0,1199.0,6.3,0.095,0.53,1890.2,44.9
|
| 45 |
+
10,mRMR,Non-search,Information theory,1155.0,1119.0,1190.0,13.7,0.656,0.593,0.5,7.0
|
| 46 |
+
11,GainRatio,Non-search,Information theory,1134.0,1100.0,1168.0,11.6,0.641,0.751,0.2,0.0
|
| 47 |
+
12,LOCO,Search,Leave-one-out,1127.0,1091.0,1165.0,5.0,0.114,0.544,246.0,12.3
|
| 48 |
+
13,SFS,Search,Forward search,1124.0,1088.0,1157.0,8.1,0.225,0.568,2771.3,48.7
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,954.0,1040.0,0.0,0.002,0.494,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,929.0,869.0,981.0,-89.9,0.689,0.758,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_all/budgets_3/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='170.6' x2='170.6' y1='16.2' y2='368.0'/><text class='sa-text' x='170.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='269.2' x2='269.2' y1='16.2' y2='368.0'/><text class='sa-text' x='269.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='367.7' x2='367.7' y1='16.2' y2='368.0'/><text class='sa-text' x='367.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='466.3' x2='466.3' y1='16.2' y2='368.0'/><text class='sa-text' x='466.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='564.8' x2='564.8' y1='16.2' y2='368.0'/><text class='sa-text' x='564.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.4' x2='663.4' y1='16.2' y2='368.0'/><text class='sa-text' x='663.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.9' x2='761.9' y1='16.2' y2='368.0'/><text class='sa-text' x='761.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='81.3' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='72.3' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='81.3' x2='847.8' y1='307.5' y2='307.5'/><text class='sa-text' x='72.3' y='313.5' text-anchor='end' font-size='15'>2.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='263.0' y2='263.0'/><text class='sa-text' x='72.3' y='269.0' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='218.4' y2='218.4'/><text class='sa-text' x='72.3' y='224.4' text-anchor='end' font-size='15'>7.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='173.9' y2='173.9'/><text class='sa-text' x='72.3' y='179.9' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='81.3' x2='847.8' y1='129.3' y2='129.3'/><text class='sa-text' x='72.3' y='135.3' text-anchor='end' font-size='15'>12.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='84.8' y2='84.8'/><text class='sa-text' x='72.3' y='90.8' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='81.3' x2='847.8' y1='40.2' y2='40.2'/><text class='sa-text' x='72.3' y='46.2' text-anchor='end' font-size='15'>17.5</text><rect class='sa-frame-line' x='81.3' y='16.2' width='766.5' height='351.8'/><text class='sa-text' x='464.5' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='123.6' y1='51.9' x2='83.3' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='96.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='116.1,352.0 356.6,108.4 485.6,96.2 531.3,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='531.3' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='485.6' cy='96.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='536.1' cy='99.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='551.8' cy='108.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='356.6' cy='108.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='700.4' cy='112.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='800.2' cy='122.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='700.3' cy='126.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='481.3' cy='144.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='498.8' cy='150.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='813.0' cy='207.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='798.7' cy='239.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='742.3' cy='263.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='116.1' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='531.3' y1='32.2' x2='409.5' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='409.5' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-text sa-halo' x='488.7' y='80.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='536.1' y1='99.7' x2='555.9' y2='85.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='555.9' y='85.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='551.8' y1='108.0' x2='601.9' y2='124.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='601.9' y='124.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='356.6' y1='108.4' x2='383.8' y2='94.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='383.8' y='94.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='700.4' y1='112.0' x2='746.2' y2='97.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='746.2' y='97.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='800.2' y1='122.7' x2='771.3' y2='139.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='771.3' y='139.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='700.3' y1='126.7' x2='670.6' y2='171.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='670.6' y='171.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><text class='sa-dim sa-halo' x='475.8' y='130.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='498.8' y1='150.7' x2='401.1' y2='164.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='401.1' y='164.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='813.0' y1='207.7' x2='828.9' y2='193.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.9' y='193.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='798.7' y1='239.7' x2='820.3' y2='225.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='820.3' y='225.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='742.3' y1='263.7' x2='708.9' y2='278.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='708.9' y='278.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='116.1' y1='352.0' x2='168.9' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='168.9' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='531.3' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 18.0% Selection time per 1k features: 45.7 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 18.0% Selection time per 1k features: 45.7 s Datasets: 71'/><circle class='sa-hit' cx='485.6' cy='96.2' r='10.5' data-tip='CART (on the Pareto front) Improvement over Random: 14.4% Selection time per 1k features: 15.7 s Datasets: 71' aria-label='CART (on the Pareto front) Improvement over Random: 14.4% Selection time per 1k features: 15.7 s Datasets: 71'/><circle class='sa-hit' cx='536.1' cy='99.7' r='10.5' data-tip='MI Improvement over Random: 14.2% Selection time per 1k features: 51.1 s Datasets: 71 Next to mRMR: 13.7%, 73.8 s' aria-label='MI Improvement over Random: 14.2% Selection time per 1k features: 51.1 s Datasets: 71 Next to mRMR: 13.7%, 73.8 s'/><circle class='sa-hit' cx='551.8' cy='108.0' r='10.5' data-tip='mRMR Improvement over Random: 13.7% Selection time per 1k features: 73.8 s Datasets: 71 Next to MI: 14.2%, 51.1 s' aria-label='mRMR Improvement over Random: 13.7% Selection time per 1k features: 73.8 s Datasets: 71 Next to MI: 14.2%, 51.1 s'/><circle class='sa-hit' cx='356.6' cy='108.4' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 13.7% Selection time per 1k features: 0.77 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 13.7% Selection time per 1k features: 0.77 s Datasets: 71'/><circle class='sa-hit' cx='700.4' cy='112.0' r='10.5' data-tip='ElasticNet Improvement over Random: 13.5% Selection time per 1k features: 2.38e+03 s Datasets: 71 Next to Lasso: 12.6%, 2.37e+03 s' aria-label='ElasticNet Improvement over Random: 13.5% Selection time per 1k features: 2.38e+03 s Datasets: 71 Next to Lasso: 12.6%, 2.37e+03 s'/><circle class='sa-hit' cx='800.2' cy='122.7' r='10.5' data-tip='(R)ReliefF Improvement over Random: 12.9% Selection time per 1k features: 2.44e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 12.9% Selection time per 1k features: 2.44e+04 s Datasets: 71'/><circle class='sa-hit' cx='700.3' cy='126.7' r='10.5' data-tip='Lasso Improvement over Random: 12.6% Selection time per 1k features: 2.37e+03 s Datasets: 71 Next to ElasticNet: 13.5%, 2.38e+03 s' aria-label='Lasso Improvement over Random: 12.6% Selection time per 1k features: 2.37e+03 s Datasets: 71 Next to ElasticNet: 13.5%, 2.38e+03 s'/><circle class='sa-hit' cx='481.3' cy='144.9' r='10.5' data-tip='GainRatio Improvement over Random: 11.6% Selection time per 1k features: 14.2 s Datasets: 71 Next to MarkovBlanket: 11.3%, 21.4 s' aria-label='GainRatio Improvement over Random: 11.6% Selection time per 1k features: 14.2 s Datasets: 71 Next to MarkovBlanket: 11.3%, 21.4 s'/><circle class='sa-hit' cx='498.8' cy='150.7' r='10.5' data-tip='MarkovBlanket Improvement over Random: 11.3% Selection time per 1k features: 21.4 s Datasets: 71 Next to GainRatio: 11.6%, 14.2 s' aria-label='MarkovBlanket Improvement over Random: 11.3% Selection time per 1k features: 21.4 s Datasets: 71 Next to GainRatio: 11.6%, 14.2 s'/><circle class='sa-hit' cx='813.0' cy='207.7' r='10.5' data-tip='SFS Improvement over Random: 8.1% Selection time per 1k features: 3.3e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 8.1% Selection time per 1k features: 3.3e+04 s Datasets: 71'/><circle class='sa-hit' cx='798.7' cy='239.7' r='10.5' data-tip='SBE Improvement over Random: 6.3% Selection time per 1k features: 2.36e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 6.3% Selection time per 1k features: 2.36e+04 s Datasets: 71'/><circle class='sa-hit' cx='742.3' cy='263.7' r='10.5' data-tip='LOCO Improvement over Random: 5.0% Selection time per 1k features: 6.32e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 5.0% Selection time per 1k features: 6.32e+03 s Datasets: 71'/><circle class='sa-hit' cx='116.1' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0028 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0028 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,17.9516,45.6523,71,True
|
| 37 |
CART,14.3581,15.705,71,True
|
| 38 |
MI,14.1607,51.1336,71,False
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='170.6' x2='170.6' y1='16.2' y2='368.0'/><text class='sa-text' x='170.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='269.2' x2='269.2' y1='16.2' y2='368.0'/><text class='sa-text' x='269.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='367.7' x2='367.7' y1='16.2' y2='368.0'/><text class='sa-text' x='367.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='466.3' x2='466.3' y1='16.2' y2='368.0'/><text class='sa-text' x='466.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='564.8' x2='564.8' y1='16.2' y2='368.0'/><text class='sa-text' x='564.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.4' x2='663.4' y1='16.2' y2='368.0'/><text class='sa-text' x='663.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.9' x2='761.9' y1='16.2' y2='368.0'/><text class='sa-text' x='761.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='81.3' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='72.3' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='81.3' x2='847.8' y1='307.5' y2='307.5'/><text class='sa-text' x='72.3' y='313.5' text-anchor='end' font-size='15'>2.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='263.0' y2='263.0'/><text class='sa-text' x='72.3' y='269.0' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='218.4' y2='218.4'/><text class='sa-text' x='72.3' y='224.4' text-anchor='end' font-size='15'>7.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='173.9' y2='173.9'/><text class='sa-text' x='72.3' y='179.9' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='81.3' x2='847.8' y1='129.3' y2='129.3'/><text class='sa-text' x='72.3' y='135.3' text-anchor='end' font-size='15'>12.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='84.8' y2='84.8'/><text class='sa-text' x='72.3' y='90.8' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='81.3' x2='847.8' y1='40.2' y2='40.2'/><text class='sa-text' x='72.3' y='46.2' text-anchor='end' font-size='15'>17.5</text><rect class='sa-frame-line' x='81.3' y='16.2' width='766.5' height='351.8'/><text class='sa-text' x='464.5' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='123.6' y1='51.9' x2='83.3' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='96.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='116.1,352.0 356.6,108.4 485.6,96.2 531.3,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='531.3' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='485.6' cy='96.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='536.1' cy='99.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='551.8' cy='108.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='356.6' cy='108.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='700.4' cy='112.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='800.2' cy='122.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='700.3' cy='126.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='481.3' cy='144.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='498.8' cy='150.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='813.0' cy='207.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='798.7' cy='239.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='742.3' cy='263.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='116.1' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='531.3' y1='32.2' x2='410.1' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='410.1' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-text sa-halo' x='487.4' y='79.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='536.1' y1='99.7' x2='557.7' y2='89.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='557.7' y='89.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='551.8' y1='108.0' x2='601.9' y2='119.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='601.9' y='119.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='356.6' y1='108.4' x2='383.8' y2='94.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='383.8' y='94.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='700.4' y1='112.0' x2='746.2' y2='97.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='746.2' y='97.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='800.2' y1='122.7' x2='770.7' y2='139.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='770.7' y='139.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='700.3' y1='126.7' x2='670.0' y2='171.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='670.0' y='171.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><text class='sa-dim sa-halo' x='469.6' y='130.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='498.8' y1='150.7' x2='403.0' y2='164.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='403.0' y='164.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='813.0' y1='207.7' x2='828.9' y2='193.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.9' y='193.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='798.7' y1='239.7' x2='820.3' y2='225.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='820.3' y='225.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='742.3' y1='263.7' x2='710.1' y2='277.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='710.1' y='277.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='116.1' y1='352.0' x2='168.9' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='168.9' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='531.3' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 18.0% Selection time per 1k features: 45.7 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 18.0% Selection time per 1k features: 45.7 s Datasets: 71'/><circle class='sa-hit' cx='485.6' cy='96.2' r='10.5' data-tip='CART (on the Pareto front) Improvement over Random: 14.4% Selection time per 1k features: 15.7 s Datasets: 71' aria-label='CART (on the Pareto front) Improvement over Random: 14.4% Selection time per 1k features: 15.7 s Datasets: 71'/><circle class='sa-hit' cx='536.1' cy='99.7' r='10.5' data-tip='MI Improvement over Random: 14.2% Selection time per 1k features: 51.1 s Datasets: 71 Next to mRMR: 13.7%, 73.8 s' aria-label='MI Improvement over Random: 14.2% Selection time per 1k features: 51.1 s Datasets: 71 Next to mRMR: 13.7%, 73.8 s'/><circle class='sa-hit' cx='551.8' cy='108.0' r='10.5' data-tip='mRMR Improvement over Random: 13.7% Selection time per 1k features: 73.8 s Datasets: 71 Next to MI: 14.2%, 51.1 s' aria-label='mRMR Improvement over Random: 13.7% Selection time per 1k features: 73.8 s Datasets: 71 Next to MI: 14.2%, 51.1 s'/><circle class='sa-hit' cx='356.6' cy='108.4' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 13.7% Selection time per 1k features: 0.77 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 13.7% Selection time per 1k features: 0.77 s Datasets: 71'/><circle class='sa-hit' cx='700.4' cy='112.0' r='10.5' data-tip='ElasticNet Improvement over Random: 13.5% Selection time per 1k features: 2.38e+03 s Datasets: 71 Next to Lasso: 12.6%, 2.37e+03 s' aria-label='ElasticNet Improvement over Random: 13.5% Selection time per 1k features: 2.38e+03 s Datasets: 71 Next to Lasso: 12.6%, 2.37e+03 s'/><circle class='sa-hit' cx='800.2' cy='122.7' r='10.5' data-tip='(R)ReliefF Improvement over Random: 12.9% Selection time per 1k features: 2.44e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 12.9% Selection time per 1k features: 2.44e+04 s Datasets: 71'/><circle class='sa-hit' cx='700.3' cy='126.7' r='10.5' data-tip='Lasso Improvement over Random: 12.6% Selection time per 1k features: 2.37e+03 s Datasets: 71 Next to ElasticNet: 13.5%, 2.38e+03 s' aria-label='Lasso Improvement over Random: 12.6% Selection time per 1k features: 2.37e+03 s Datasets: 71 Next to ElasticNet: 13.5%, 2.38e+03 s'/><circle class='sa-hit' cx='481.3' cy='144.9' r='10.5' data-tip='GainRatio Improvement over Random: 11.6% Selection time per 1k features: 14.2 s Datasets: 71 Next to MarkovBlanket: 11.3%, 21.4 s' aria-label='GainRatio Improvement over Random: 11.6% Selection time per 1k features: 14.2 s Datasets: 71 Next to MarkovBlanket: 11.3%, 21.4 s'/><circle class='sa-hit' cx='498.8' cy='150.7' r='10.5' data-tip='MarkovBlanket Improvement over Random: 11.3% Selection time per 1k features: 21.4 s Datasets: 71 Next to GainRatio: 11.6%, 14.2 s' aria-label='MarkovBlanket Improvement over Random: 11.3% Selection time per 1k features: 21.4 s Datasets: 71 Next to GainRatio: 11.6%, 14.2 s'/><circle class='sa-hit' cx='813.0' cy='207.7' r='10.5' data-tip='SFS Improvement over Random: 8.1% Selection time per 1k features: 3.3e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 8.1% Selection time per 1k features: 3.3e+04 s Datasets: 71'/><circle class='sa-hit' cx='798.7' cy='239.7' r='10.5' data-tip='SBE Improvement over Random: 6.3% Selection time per 1k features: 2.36e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 6.3% Selection time per 1k features: 2.36e+04 s Datasets: 71'/><circle class='sa-hit' cx='742.3' cy='263.7' r='10.5' data-tip='LOCO Improvement over Random: 5.0% Selection time per 1k features: 6.32e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 5.0% Selection time per 1k features: 6.32e+03 s Datasets: 71'/><circle class='sa-hit' cx='116.1' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0028 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0028 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,17.9516,45.6523,71,True
|
| 37 |
CART,14.3581,15.705,71,True
|
| 38 |
MI,14.1607,51.1336,71,False
|
data/subsets/tasks_all/models_all/budgets_3/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1288.0,1248.0,1331.0,5.9,65.0,12.4,,0.661,0.6,0.2,0.0,71,1065
|
| 3 |
-
2,CART,Non-search,Tree,1232.0,1201.0,1272.0,6.65,59.7,15.6,,0.402,0.507,0.1,0.0,71,1065
|
| 4 |
-
3,MI,Non-search,Information theory,1218.0,1183.0,1256.0,6.94,57.6,15.8,,0.649,0.723,0.4,0.0,71,1065
|
| 5 |
-
4,ElasticNet,Non-search,Regularization,1212.0,1171.0,1251.0,7.02,57.0,15.9,,0.419,0.665,7.6,0.0,71,1065
|
| 6 |
-
5,Lasso,Non-search,Regularization,1192.0,1154.0,1230.0,7.27,55.2,16.4,,0.334,0.643,2.0,0.0,71,1065
|
| 7 |
-
6,F-Test,Non-search,Correlation,1189.0,1157.0,1222.0,7.37,54.5,15.9,,0.668,0.781,0.0,0.0,71,1065
|
| 8 |
-
7,(R)ReliefF,Non-search,Distance,1172.0,1137.0,1210.0,7.8,51.4,17.0,,0.656,0.667,14.3,0.0,71,1065
|
| 9 |
-
8,MarkovBlanket,Search,Backward search,1158.0,1122.0,1189.0,8.15,48.9,18.6,,0.434,0.473,0.1,0.0,71,1065
|
| 10 |
-
9,SBE,Search,Backward search,1155.0,1106.0,1199.0,8.03,49.8,19.9,,0.095,0.53,1890.2,44.9,71,1065
|
| 11 |
-
10,mRMR,Non-search,Information theory,1155.0,1119.0,1190.0,8.09,49.4,17.2,,0.656,0.593,0.5,7.0,71,1065
|
| 12 |
-
11,GainRatio,Non-search,Information theory,1134.0,1100.0,1168.0,8.44,46.9,18.3,,0.641,0.751,0.2,0.0,71,1065
|
| 13 |
-
12,LOCO,Search,Leave-one-out,1127.0,1091.0,1165.0,8.21,48.5,20.4,,0.114,0.544,246.0,12.3,71,1065
|
| 14 |
-
13,SFS,Search,Forward search,1124.0,1088.0,1157.0,8.32,47.7,20.2,,0.225,0.568,2771.3,48.7,71,1065
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,954.0,1040.0,10.09,35.1,25.9,,0.002,0.494,0.0,0.0,71,1065
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,929.0,869.0,981.0,11.73,23.4,33.1,,0.689,0.758,0.2,0.0,71,1065
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1288.0,1248.0,1331.0,5.9,65.0,12.4,18.0,,0.661,0.6,0.2,0.0,71,1065
|
| 3 |
+
2,CART,Non-search,Tree,1232.0,1201.0,1272.0,6.65,59.7,15.6,14.4,,0.402,0.507,0.1,0.0,71,1065
|
| 4 |
+
3,MI,Non-search,Information theory,1218.0,1183.0,1256.0,6.94,57.6,15.8,14.2,,0.649,0.723,0.4,0.0,71,1065
|
| 5 |
+
4,ElasticNet,Non-search,Regularization,1212.0,1171.0,1251.0,7.02,57.0,15.9,13.5,,0.419,0.665,7.6,0.0,71,1065
|
| 6 |
+
5,Lasso,Non-search,Regularization,1192.0,1154.0,1230.0,7.27,55.2,16.4,12.6,,0.334,0.643,2.0,0.0,71,1065
|
| 7 |
+
6,F-Test,Non-search,Correlation,1189.0,1157.0,1222.0,7.37,54.5,15.9,13.7,,0.668,0.781,0.0,0.0,71,1065
|
| 8 |
+
7,(R)ReliefF,Non-search,Distance,1172.0,1137.0,1210.0,7.8,51.4,17.0,12.9,,0.656,0.667,14.3,0.0,71,1065
|
| 9 |
+
8,MarkovBlanket,Search,Backward search,1158.0,1122.0,1189.0,8.15,48.9,18.6,11.3,,0.434,0.473,0.1,0.0,71,1065
|
| 10 |
+
9,SBE,Search,Backward search,1155.0,1106.0,1199.0,8.03,49.8,19.9,6.3,,0.095,0.53,1890.2,44.9,71,1065
|
| 11 |
+
10,mRMR,Non-search,Information theory,1155.0,1119.0,1190.0,8.09,49.4,17.2,13.7,,0.656,0.593,0.5,7.0,71,1065
|
| 12 |
+
11,GainRatio,Non-search,Information theory,1134.0,1100.0,1168.0,8.44,46.9,18.3,11.6,,0.641,0.751,0.2,0.0,71,1065
|
| 13 |
+
12,LOCO,Search,Leave-one-out,1127.0,1091.0,1165.0,8.21,48.5,20.4,5.0,,0.114,0.544,246.0,12.3,71,1065
|
| 14 |
+
13,SFS,Search,Forward search,1124.0,1088.0,1157.0,8.32,47.7,20.2,8.1,,0.225,0.568,2771.3,48.7,71,1065
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,954.0,1040.0,10.09,35.1,25.9,0.0,,0.002,0.494,0.0,0.0,71,1065
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,929.0,869.0,981.0,11.73,23.4,33.1,-89.9,,0.689,0.758,0.2,0.0,71,1065
|
data/subsets/tasks_all/models_all/budgets_4/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1250.0'>1250<span class='sub'>+34 / −29</span></td><td class='num' data-sort='6.2'>6.20</td><td class='num' data-sort='10.7'>10.7</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+35 / −34</span></td><td class='num' data-sort='6.97'>6.97</td><td class='num' data-sort='12.6'>12.6</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+34 / −34</span></td><td class='num' data-sort='7.21'>7.21</td><td class='num' data-sort='12.4'>12.4</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='7.7'>7.7</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+29 / −34</span></td><td class='num' data-sort='7.24'>7.24</td><td class='num' data-sort='12.6'>12.6</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+38 / −39</span></td><td class='num' data-sort='7.39'>7.39</td><td class='num' data-sort='11.8'>11.8</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1165.0'>1165<span class='sub'>+38 / −35</span></td><td class='num' data-sort='7.67'>7.67</td><td class='num' data-sort='13.0'>13.0</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='14.4'>14.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1140.0'>1140<span class='sub'>+36 / −33</span></td><td class='num' data-sort='7.56'>7.56</td><td class='num' data-sort='13.1'>13.1</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1131.0'>1131<span class='sub'>+34 / −36</span></td><td class='num' data-sort='7.89'>7.89</td><td class='num' data-sort='12.5'>12.5</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.6'>0.6</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1115.0'>1115<span class='sub'>+36 / −38</span></td><td class='num' data-sort='8.26'>8.26</td><td class='num' data-sort='14.0'>14.0</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1113.0'>1113<span class='sub'>+37 / −40</span></td><td class='num' data-sort='8.31'>8.31</td><td class='num' data-sort='15.3'>15.3</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='989.9'>989.9</td><td class='num' data-sort='41.5'>41.5</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+33 / −32</span></td><td class='num' data-sort='8.26'>8.26</td><td class='num' data-sort='15.5'>15.5</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='243.3'>243.3</td><td class='num' data-sort='18.7'>18.7</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+35 / −33</span></td><td class='num' data-sort='8.5'>8.50</td><td class='num' data-sort='14.9'>14.9</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1092.0'>1092<span class='sub'>+30 / −33</span></td><td class='num' data-sort='8.46'>8.46</td><td class='num' data-sort='15.4'>15.4</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='2953.5'>2,953.5</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+40 / −42</span></td><td class='num' data-sort='9.67'>9.67</td><td class='num' data-sort='18.9'>18.9</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='980.0'>980<span class='sub'>+46 / −56</span></td><td class='num' data-sort='10.39'>10.39</td><td class='num' data-sort='23.1'>23.1</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1250.0,1221.0,1284.0,
|
| 37 |
-
2,CART,Non-search,Tree,1189.0,1155.0,1224.0,
|
| 38 |
-
3,ElasticNet,Non-search,Regularization,1172.0,1138.0,1206.0,
|
| 39 |
-
4,MI,Non-search,Information theory,1171.0,1137.0,1200.0,7.
|
| 40 |
-
5,F-Test,Non-search,Correlation,1168.0,1129.0,1206.0,
|
| 41 |
-
6,(R)ReliefF,Non-search,Distance,1165.0,1130.0,1203.0,
|
| 42 |
-
7,Lasso,Non-search,Regularization,1140.0,1107.0,1176.0,7.
|
| 43 |
-
8,mRMR,Non-search,Information theory,1131.0,1095.0,1165.0,
|
| 44 |
-
9,MarkovBlanket,Search,Backward search,1115.0,1077.0,1151.0,
|
| 45 |
-
10,SBE,Search,Backward search,1113.0,1073.0,1150.0,
|
| 46 |
-
11,LOCO,Search,Leave-one-out,1097.0,1065.0,1130.0,
|
| 47 |
-
12,GainRatio,Non-search,Information theory,1097.0,1064.0,1132.0,
|
| 48 |
-
13,SFS,Search,Forward search,1092.0,1059.0,1122.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,958.0,1040.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,980.0,924.0,1026.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1250.0'>1250<span class='sub'>+34 / −29</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+35 / −34</span></td><td class='num' data-sort='7.6'>7.6</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+34 / −34</span></td><td class='num' data-sort='8.7'>8.7</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='7.7'>7.7</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+29 / −34</span></td><td class='num' data-sort='7.4'>7.4</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+38 / −39</span></td><td class='num' data-sort='9.9'>9.9</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1165.0'>1165<span class='sub'>+38 / −35</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='14.4'>14.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1140.0'>1140<span class='sub'>+36 / −33</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1131.0'>1131<span class='sub'>+34 / −36</span></td><td class='num' data-sort='9.0'>9.0</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.6'>0.6</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1115.0'>1115<span class='sub'>+36 / −38</span></td><td class='num' data-sort='6.5'>6.5</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1113.0'>1113<span class='sub'>+37 / −40</span></td><td class='num' data-sort='3.9'>3.9</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='989.9'>989.9</td><td class='num' data-sort='41.5'>41.5</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+33 / −32</span></td><td class='num' data-sort='4.4'>4.4</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='243.3'>243.3</td><td class='num' data-sort='18.7'>18.7</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+35 / −33</span></td><td class='num' data-sort='6.0'>6.0</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1092.0'>1092<span class='sub'>+30 / −33</span></td><td class='num' data-sort='3.4'>3.4</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='2953.5'>2,953.5</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+40 / −42</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='980.0'>980<span class='sub'>+46 / −56</span></td><td class='num' data-sort='-92.1'>-92.1</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1250.0,1221.0,1284.0,11.1,0.597,0.507,0.2,0.0
|
| 37 |
+
2,CART,Non-search,Tree,1189.0,1155.0,1224.0,7.6,0.324,0.482,0.1,0.0
|
| 38 |
+
3,ElasticNet,Non-search,Regularization,1172.0,1138.0,1206.0,8.7,0.308,0.568,7.7,0.0
|
| 39 |
+
4,MI,Non-search,Information theory,1171.0,1137.0,1200.0,7.4,0.587,0.627,0.4,0.0
|
| 40 |
+
5,F-Test,Non-search,Correlation,1168.0,1129.0,1206.0,9.9,0.562,0.689,0.0,0.0
|
| 41 |
+
6,(R)ReliefF,Non-search,Distance,1165.0,1130.0,1203.0,8.1,0.612,0.594,14.4,0.0
|
| 42 |
+
7,Lasso,Non-search,Regularization,1140.0,1107.0,1176.0,7.5,0.246,0.563,2.0,0.0
|
| 43 |
+
8,mRMR,Non-search,Information theory,1131.0,1095.0,1165.0,9.0,0.651,0.53,0.6,9.9
|
| 44 |
+
9,MarkovBlanket,Search,Backward search,1115.0,1077.0,1151.0,6.5,0.404,0.463,0.1,0.0
|
| 45 |
+
10,SBE,Search,Backward search,1113.0,1073.0,1150.0,3.9,0.059,0.517,989.9,41.5
|
| 46 |
+
11,LOCO,Search,Leave-one-out,1097.0,1065.0,1130.0,4.4,0.08,0.52,243.3,18.7
|
| 47 |
+
12,GainRatio,Non-search,Information theory,1097.0,1064.0,1132.0,6.0,0.55,0.659,0.2,0.0
|
| 48 |
+
13,SFS,Search,Forward search,1092.0,1059.0,1122.0,3.4,0.176,0.525,2953.5,48.8
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,958.0,1040.0,0.0,0.002,0.5,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,980.0,924.0,1026.0,-92.1,0.678,0.701,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_all/budgets_4/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='106.3' x2='106.3' y1='16.2' y2='368.0'/><text class='sa-text' x='106.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='214.2' x2='214.2' y1='16.2' y2='368.0'/><text class='sa-text' x='214.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='322.2' x2='322.2' y1='16.2' y2='368.0'/><text class='sa-text' x='322.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='430.1' x2='430.1' y1='16.2' y2='368.0'/><text class='sa-text' x='430.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='538.0' x2='538.0' y1='16.2' y2='368.0'/><text class='sa-text' x='538.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='645.9' x2='645.9' y1='16.2' y2='368.0'/><text class='sa-text' x='645.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='753.8' x2='753.8' y1='16.2' y2='368.0'/><text class='sa-text' x='753.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='294.5' y2='294.5'/><text class='sa-text' x='57.9' y='300.5' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='236.9' y2='236.9'/><text class='sa-text' x='57.9' y='242.9' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='179.3' y2='179.3'/><text class='sa-text' x='57.9' y='185.3' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='121.7' y2='121.7'/><text class='sa-text' x='57.9' y='127.7' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='64.2' y2='64.2'/><text class='sa-text' x='57.9' y='70.2' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 314.7,65.8 500.7,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='500.7' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='314.7' cy='65.8' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='549.3' cy='92.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.2' cy='102.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.8' cy='120.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='451.4' cy='133.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.6' cy='137.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='506.8' cy='138.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='463.5' cy='163.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='446.4' cy='178.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='731.9' cy='226.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='775.2' cy='240.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='253.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='500.7' y1='32.2' x2='394.3' y2='46.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='394.3' y='46.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='314.7' y1='65.8' x2='275.0' y2='80.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='275.0' y='80.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='549.3' y1='92.6' x2='577.0' y2='78.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='577.0' y='78.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='686.2' y1='102.8' x2='732.7' y2='88.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='732.7' y='88.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='796.9' y='133.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='451.4' y1='133.6' x2='470.3' y2='118.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='470.3' y='118.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='686.6' y1='137.3' x2='746.5' y2='165.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='746.5' y='165.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='506.8' y1='138.8' x2='534.5' y2='125.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='534.5' y='125.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='463.5' y1='163.8' x2='582.2' y2='153.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='582.2' y='153.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='446.4' y1='178.9' x2='382.2' y2='192.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='382.2' y='192.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='731.9' y1='226.8' x2='755.2' y2='212.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='755.2' y='212.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='775.2' y1='240.1' x2='748.1' y2='253.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='748.1' y='253.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='812.3' y1='253.9' x2='828.6' y2='267.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='267.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='500.7' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 45.1 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 45.1 s Datasets: 71'/><circle class='sa-hit' cx='314.7' cy='65.8' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 9.9% Selection time per 1k features: 0.852 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 9.9% Selection time per 1k features: 0.852 s Datasets: 71'/><circle class='sa-hit' cx='549.3' cy='92.6' r='10.5' data-tip='mRMR Improvement over Random: 9.0% Selection time per 1k features: 127 s Datasets: 71' aria-label='mRMR Improvement over Random: 9.0% Selection time per 1k features: 127 s Datasets: 71'/><circle class='sa-hit' cx='686.2' cy='102.8' r='10.5' data-tip='ElasticNet Improvement over Random: 8.7% Selection time per 1k features: 2.36e+03 s Datasets: 71' aria-label='ElasticNet Improvement over Random: 8.7% Selection time per 1k features: 2.36e+03 s Datasets: 71'/><circle class='sa-hit' cx='795.8' cy='120.1' r='10.5' data-tip='(R)ReliefF Improvement over Random: 8.1% Selection time per 1k features: 2.45e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 8.1% Selection time per 1k features: 2.45e+04 s Datasets: 71'/><circle class='sa-hit' cx='451.4' cy='133.6' r='10.5' data-tip='CART Improvement over Random: 7.6% Selection time per 1k features: 15.8 s Datasets: 71' aria-label='CART Improvement over Random: 7.6% Selection time per 1k features: 15.8 s Datasets: 71'/><circle class='sa-hit' cx='686.6' cy='137.3' r='10.5' data-tip='Lasso Improvement over Random: 7.5% Selection time per 1k features: 2.38e+03 s Datasets: 71' aria-label='Lasso Improvement over Random: 7.5% Selection time per 1k features: 2.38e+03 s Datasets: 71'/><circle class='sa-hit' cx='506.8' cy='138.8' r='10.5' data-tip='MI Improvement over Random: 7.4% Selection time per 1k features: 51.3 s Datasets: 71' aria-label='MI Improvement over Random: 7.4% Selection time per 1k features: 51.3 s Datasets: 71'/><circle class='sa-hit' cx='463.5' cy='163.8' r='10.5' data-tip='MarkovBlanket Improvement over Random: 6.5% Selection time per 1k features: 20.4 s Datasets: 71' aria-label='MarkovBlanket Improvement over Random: 6.5% Selection time per 1k features: 20.4 s Datasets: 71'/><circle class='sa-hit' cx='446.4' cy='178.9' r='10.5' data-tip='GainRatio Improvement over Random: 6.0% Selection time per 1k features: 14.2 s Datasets: 71' aria-label='GainRatio Improvement over Random: 6.0% Selection time per 1k features: 14.2 s Datasets: 71'/><circle class='sa-hit' cx='731.9' cy='226.8' r='10.5' data-tip='LOCO Improvement over Random: 4.4% Selection time per 1k features: 6.27e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 4.4% Selection time per 1k features: 6.27e+03 s Datasets: 71'/><circle class='sa-hit' cx='775.2' cy='240.1' r='10.5' data-tip='SBE Improvement over Random: 3.9% Selection time per 1k features: 1.58e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 3.9% Selection time per 1k features: 1.58e+04 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='253.9' r='10.5' data-tip='SFS Improvement over Random: 3.4% Selection time per 1k features: 3.48e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 3.4% Selection time per 1k features: 3.48e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0092 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0092 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,11.1111,45.1078,71,True
|
| 37 |
F-Test,9.9441,0.8521,71,True
|
| 38 |
mRMR,9.0137,127.2752,71,False
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='106.3' x2='106.3' y1='16.2' y2='368.0'/><text class='sa-text' x='106.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='214.2' x2='214.2' y1='16.2' y2='368.0'/><text class='sa-text' x='214.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='322.2' x2='322.2' y1='16.2' y2='368.0'/><text class='sa-text' x='322.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='430.1' x2='430.1' y1='16.2' y2='368.0'/><text class='sa-text' x='430.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='538.0' x2='538.0' y1='16.2' y2='368.0'/><text class='sa-text' x='538.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='645.9' x2='645.9' y1='16.2' y2='368.0'/><text class='sa-text' x='645.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='753.8' x2='753.8' y1='16.2' y2='368.0'/><text class='sa-text' x='753.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='294.5' y2='294.5'/><text class='sa-text' x='57.9' y='300.5' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='236.9' y2='236.9'/><text class='sa-text' x='57.9' y='242.9' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='179.3' y2='179.3'/><text class='sa-text' x='57.9' y='185.3' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='121.7' y2='121.7'/><text class='sa-text' x='57.9' y='127.7' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='64.2' y2='64.2'/><text class='sa-text' x='57.9' y='70.2' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 314.7,65.8 500.7,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='500.7' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='314.7' cy='65.8' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='549.3' cy='92.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.2' cy='102.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.8' cy='120.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='451.4' cy='133.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.6' cy='137.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='506.8' cy='138.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='463.5' cy='163.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='446.4' cy='178.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='731.9' cy='226.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='775.2' cy='240.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='253.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='500.7' y1='32.2' x2='394.3' y2='46.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='394.3' y='46.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='314.7' y1='65.8' x2='275.0' y2='80.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='275.0' y='80.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='549.3' y1='92.6' x2='577.0' y2='78.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='577.0' y='78.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='686.2' y1='102.8' x2='732.7' y2='88.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='732.7' y='88.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='796.9' y='133.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='451.4' y1='133.6' x2='472.8' y2='118.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='472.8' y='118.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='686.6' y1='137.3' x2='744.0' y2='163.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='744.0' y='163.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='506.8' y1='138.8' x2='537.6' y2='120.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='537.6' y='120.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='463.5' y1='163.8' x2='579.7' y2='156.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='579.7' y='156.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='446.4' y1='178.9' x2='382.2' y2='192.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='382.2' y='192.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='731.9' y1='226.8' x2='755.2' y2='212.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='755.2' y='212.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='775.2' y1='240.1' x2='748.1' y2='253.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='748.1' y='253.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='812.3' y1='253.9' x2='828.6' y2='267.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='267.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='500.7' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 45.1 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 45.1 s Datasets: 71'/><circle class='sa-hit' cx='314.7' cy='65.8' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 9.9% Selection time per 1k features: 0.852 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 9.9% Selection time per 1k features: 0.852 s Datasets: 71'/><circle class='sa-hit' cx='549.3' cy='92.6' r='10.5' data-tip='mRMR Improvement over Random: 9.0% Selection time per 1k features: 127 s Datasets: 71' aria-label='mRMR Improvement over Random: 9.0% Selection time per 1k features: 127 s Datasets: 71'/><circle class='sa-hit' cx='686.2' cy='102.8' r='10.5' data-tip='ElasticNet Improvement over Random: 8.7% Selection time per 1k features: 2.36e+03 s Datasets: 71' aria-label='ElasticNet Improvement over Random: 8.7% Selection time per 1k features: 2.36e+03 s Datasets: 71'/><circle class='sa-hit' cx='795.8' cy='120.1' r='10.5' data-tip='(R)ReliefF Improvement over Random: 8.1% Selection time per 1k features: 2.45e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 8.1% Selection time per 1k features: 2.45e+04 s Datasets: 71'/><circle class='sa-hit' cx='451.4' cy='133.6' r='10.5' data-tip='CART Improvement over Random: 7.6% Selection time per 1k features: 15.8 s Datasets: 71' aria-label='CART Improvement over Random: 7.6% Selection time per 1k features: 15.8 s Datasets: 71'/><circle class='sa-hit' cx='686.6' cy='137.3' r='10.5' data-tip='Lasso Improvement over Random: 7.5% Selection time per 1k features: 2.38e+03 s Datasets: 71' aria-label='Lasso Improvement over Random: 7.5% Selection time per 1k features: 2.38e+03 s Datasets: 71'/><circle class='sa-hit' cx='506.8' cy='138.8' r='10.5' data-tip='MI Improvement over Random: 7.4% Selection time per 1k features: 51.3 s Datasets: 71' aria-label='MI Improvement over Random: 7.4% Selection time per 1k features: 51.3 s Datasets: 71'/><circle class='sa-hit' cx='463.5' cy='163.8' r='10.5' data-tip='MarkovBlanket Improvement over Random: 6.5% Selection time per 1k features: 20.4 s Datasets: 71' aria-label='MarkovBlanket Improvement over Random: 6.5% Selection time per 1k features: 20.4 s Datasets: 71'/><circle class='sa-hit' cx='446.4' cy='178.9' r='10.5' data-tip='GainRatio Improvement over Random: 6.0% Selection time per 1k features: 14.2 s Datasets: 71' aria-label='GainRatio Improvement over Random: 6.0% Selection time per 1k features: 14.2 s Datasets: 71'/><circle class='sa-hit' cx='731.9' cy='226.8' r='10.5' data-tip='LOCO Improvement over Random: 4.4% Selection time per 1k features: 6.27e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 4.4% Selection time per 1k features: 6.27e+03 s Datasets: 71'/><circle class='sa-hit' cx='775.2' cy='240.1' r='10.5' data-tip='SBE Improvement over Random: 3.9% Selection time per 1k features: 1.58e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 3.9% Selection time per 1k features: 1.58e+04 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='253.9' r='10.5' data-tip='SFS Improvement over Random: 3.4% Selection time per 1k features: 3.48e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 3.4% Selection time per 1k features: 3.48e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0092 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0092 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,11.1111,45.1078,71,True
|
| 37 |
F-Test,9.9441,0.8521,71,True
|
| 38 |
mRMR,9.0137,127.2752,71,False
|
data/subsets/tasks_all/models_all/budgets_4/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1250.0,1221.0,1284.0,6.2,62.9,10.7,,0.597,0.507,0.2,0.0,71,1065
|
| 3 |
-
2,CART,Non-search,Tree,1189.0,1155.0,1224.0,6.97,57.4,12.6,,0.324,0.482,0.1,0.0,71,1065
|
| 4 |
-
3,ElasticNet,Non-search,Regularization,1172.0,1138.0,1206.0,7.21,55.6,12.4,,0.308,0.568,7.7,0.0,71,1065
|
| 5 |
-
4,MI,Non-search,Information theory,1171.0,1137.0,1200.0,7.24,55.4,12.6,,0.587,0.627,0.4,0.0,71,1065
|
| 6 |
-
5,F-Test,Non-search,Correlation,1168.0,1129.0,1206.0,7.39,54.3,11.8,,0.562,0.689,0.0,0.0,71,1065
|
| 7 |
-
6,(R)ReliefF,Non-search,Distance,1165.0,1130.0,1203.0,7.67,52.3,13.0,,0.612,0.594,14.4,0.0,71,1065
|
| 8 |
-
7,Lasso,Non-search,Regularization,1140.0,1107.0,1176.0,7.56,53.1,13.1,,0.246,0.563,2.0,0.0,71,1065
|
| 9 |
-
8,mRMR,Non-search,Information theory,1131.0,1095.0,1165.0,7.89,50.8,12.5,,0.651,0.53,0.6,9.9,71,1065
|
| 10 |
-
9,MarkovBlanket,Search,Backward search,1115.0,1077.0,1151.0,8.26,48.1,14.0,,0.404,0.463,0.1,0.0,71,1065
|
| 11 |
-
10,SBE,Search,Backward search,1113.0,1073.0,1150.0,8.31,47.8,15.3,,0.059,0.517,989.9,41.5,71,1064
|
| 12 |
-
11,LOCO,Search,Leave-one-out,1097.0,1065.0,1130.0,8.26,48.2,15.5,,0.08,0.52,243.3,18.7,71,1065
|
| 13 |
-
12,GainRatio,Non-search,Information theory,1097.0,1064.0,1132.0,8.5,46.4,14.9,,0.55,0.659,0.2,0.0,71,1065
|
| 14 |
-
13,SFS,Search,Forward search,1092.0,1059.0,1122.0,8.46,46.7,15.4,,0.176,0.525,2953.5,48.8,71,1064
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,958.0,1040.0,9.67,38.1,18.9,,0.002,0.5,0.0,0.0,71,1065
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,980.0,924.0,1026.0,10.39,32.9,23.1,,0.678,0.701,0.2,0.0,71,1065
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1250.0,1221.0,1284.0,6.2,62.9,10.7,11.1,,0.597,0.507,0.2,0.0,71,1065
|
| 3 |
+
2,CART,Non-search,Tree,1189.0,1155.0,1224.0,6.97,57.4,12.6,7.6,,0.324,0.482,0.1,0.0,71,1065
|
| 4 |
+
3,ElasticNet,Non-search,Regularization,1172.0,1138.0,1206.0,7.21,55.6,12.4,8.7,,0.308,0.568,7.7,0.0,71,1065
|
| 5 |
+
4,MI,Non-search,Information theory,1171.0,1137.0,1200.0,7.24,55.4,12.6,7.4,,0.587,0.627,0.4,0.0,71,1065
|
| 6 |
+
5,F-Test,Non-search,Correlation,1168.0,1129.0,1206.0,7.39,54.3,11.8,9.9,,0.562,0.689,0.0,0.0,71,1065
|
| 7 |
+
6,(R)ReliefF,Non-search,Distance,1165.0,1130.0,1203.0,7.67,52.3,13.0,8.1,,0.612,0.594,14.4,0.0,71,1065
|
| 8 |
+
7,Lasso,Non-search,Regularization,1140.0,1107.0,1176.0,7.56,53.1,13.1,7.5,,0.246,0.563,2.0,0.0,71,1065
|
| 9 |
+
8,mRMR,Non-search,Information theory,1131.0,1095.0,1165.0,7.89,50.8,12.5,9.0,,0.651,0.53,0.6,9.9,71,1065
|
| 10 |
+
9,MarkovBlanket,Search,Backward search,1115.0,1077.0,1151.0,8.26,48.1,14.0,6.5,,0.404,0.463,0.1,0.0,71,1065
|
| 11 |
+
10,SBE,Search,Backward search,1113.0,1073.0,1150.0,8.31,47.8,15.3,3.9,,0.059,0.517,989.9,41.5,71,1064
|
| 12 |
+
11,LOCO,Search,Leave-one-out,1097.0,1065.0,1130.0,8.26,48.2,15.5,4.4,,0.08,0.52,243.3,18.7,71,1065
|
| 13 |
+
12,GainRatio,Non-search,Information theory,1097.0,1064.0,1132.0,8.5,46.4,14.9,6.0,,0.55,0.659,0.2,0.0,71,1065
|
| 14 |
+
13,SFS,Search,Forward search,1092.0,1059.0,1122.0,8.46,46.7,15.4,3.4,,0.176,0.525,2953.5,48.8,71,1064
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,958.0,1040.0,9.67,38.1,18.9,0.0,,0.002,0.5,0.0,0.0,71,1065
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,980.0,924.0,1026.0,10.39,32.9,23.1,-92.1,,0.678,0.701,0.2,0.0,71,1065
|
data/subsets/tasks_all/models_all/budgets_all/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1310.0'>1310<span class='sub'>+32 / −31</span></td><td class='num' data-sort='6.15'>6.15</td><td class='num' data-sort='15.1'>15.1</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1265.0'>1265<span class='sub'>+34 / −29</span></td><td class='num' data-sort='6.69'>6.69</td><td class='num' data-sort='18.0'>18.0</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+32 / −33</span></td><td class='num' data-sort='6.87'>6.87</td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1248.0'>1248<span class='sub'>+28 / −24</span></td><td class='num' data-sort='6.99'>6.99</td><td class='num' data-sort='18.2'>18.2</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1229.0'>1229<span class='sub'>+28 / −29</span></td><td class='num' data-sort='7.35'>7.35</td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1226.0'>1226<span class='sub'>+29 / −32</span></td><td class='num' data-sort='7.2'>7.20</td><td class='num' data-sort='18.7'>18.7</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='2.1'>2.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1220.0'>1220<span class='sub'>+27 / −28</span></td><td class='num' data-sort='7.55'>7.55</td><td class='num' data-sort='19.6'>19.6</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1198.0'>1198<span class='sub'>+32 / −32</span></td><td class='num' data-sort='8.0'>8.00</td><td class='num' data-sort='20.6'>20.6</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1193.0'>1193<span class='sub'>+27 / −28</span></td><td class='num' data-sort='7.96'>7.96</td><td class='num' data-sort='19.9'>19.9</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1183.0'>1183<span class='sub'>+28 / −26</span></td><td class='num' data-sort='8.19'>8.19</td><td class='num' data-sort='21.5'>21.5</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1178.0'>1178<span class='sub'>+31 / −32</span></td><td class='num' data-sort='7.99'>7.99</td><td class='num' data-sort='21.7'>21.7</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2105.8'>2,105.8</td><td class='num' data-sort='43.5'>43.5</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+38 / −39</span></td><td class='num' data-sort='8.31'>8.31</td><td class='num' data-sort='23.7'>23.7</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2495.5'>2,495.5</td><td class='num' data-sort='46.5'>46.5</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1151.0'>1151<span class='sub'>+33 / −32</span></td><td class='num' data-sort='8.39'>8.39</td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='241.5'>241.5</td><td class='num' data-sort='7.6'>7.6</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+32 / −34</span></td><td class='num' data-sort='10.72'>10.72</td><td class='num' data-sort='32.0'>32.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='965.0'>965<span class='sub'>+43 / −52</span></td><td class='num' data-sort='11.64'>11.64</td><td class='num' data-sort='36.6'>36.6</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1310.0,1279.0,1342.0,
|
| 37 |
-
2,CART,Non-search,Tree,1265.0,1236.0,1299.0,
|
| 38 |
-
3,ElasticNet,Non-search,Regularization,1252.0,1219.0,1284.0,
|
| 39 |
-
4,MI,Non-search,Information theory,1248.0,1224.0,1276.0,
|
| 40 |
-
5,F-Test,Non-search,Correlation,1229.0,1200.0,1257.0,
|
| 41 |
-
6,Lasso,Non-search,Regularization,1226.0,1194.0,1255.0,
|
| 42 |
-
7,MarkovBlanket,Search,Backward search,1220.0,1192.0,1247.0,
|
| 43 |
-
8,(R)ReliefF,Non-search,Distance,1198.0,1166.0,1230.0,
|
| 44 |
-
9,mRMR,Non-search,Information theory,1193.0,1165.0,1220.0,
|
| 45 |
-
10,GainRatio,Non-search,Information theory,1183.0,1157.0,1211.0,
|
| 46 |
-
11,SFS,Search,Forward search,1178.0,1146.0,1209.0,
|
| 47 |
-
12,SBE,Search,Backward search,1169.0,1130.0,1207.0,
|
| 48 |
-
13,LOCO,Search,Leave-one-out,1151.0,1119.0,1184.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,966.0,1032.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,965.0,913.0,1008.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1310.0'>1310<span class='sub'>+32 / −31</span></td><td class='num' data-sort='26.2'>26.2</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1265.0'>1265<span class='sub'>+34 / −29</span></td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+32 / −33</span></td><td class='num' data-sort='21.9'>21.9</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='7.6'>7.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1248.0'>1248<span class='sub'>+28 / −24</span></td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1229.0'>1229<span class='sub'>+28 / −29</span></td><td class='num' data-sort='22.8'>22.8</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1226.0'>1226<span class='sub'>+29 / −32</span></td><td class='num' data-sort='20.9'>20.9</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='2.1'>2.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1220.0'>1220<span class='sub'>+27 / −28</span></td><td class='num' data-sort='21.2'>21.2</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1198.0'>1198<span class='sub'>+32 / −32</span></td><td class='num' data-sort='18.6'>18.6</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='14.3'>14.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1193.0'>1193<span class='sub'>+27 / −28</span></td><td class='num' data-sort='21.7'>21.7</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1183.0'>1183<span class='sub'>+28 / −26</span></td><td class='num' data-sort='19.5'>19.5</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1178.0'>1178<span class='sub'>+31 / −32</span></td><td class='num' data-sort='18.2'>18.2</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2105.8'>2,105.8</td><td class='num' data-sort='43.5'>43.5</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+38 / −39</span></td><td class='num' data-sort='11.4'>11.4</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2495.5'>2,495.5</td><td class='num' data-sort='46.5'>46.5</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1151.0'>1151<span class='sub'>+33 / −32</span></td><td class='num' data-sort='13.0'>13.0</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='241.5'>241.5</td><td class='num' data-sort='7.6'>7.6</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+32 / −34</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='965.0'>965<span class='sub'>+43 / −52</span></td><td class='num' data-sort='-18.6'>-18.6</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1310.0,1279.0,1342.0,26.2,0.643,0.669,0.2,0.0
|
| 37 |
+
2,CART,Non-search,Tree,1265.0,1236.0,1299.0,23.8,0.422,0.586,0.1,0.0
|
| 38 |
+
3,ElasticNet,Non-search,Regularization,1252.0,1219.0,1284.0,21.9,0.447,0.76,7.6,0.0
|
| 39 |
+
4,MI,Non-search,Information theory,1248.0,1224.0,1276.0,23.8,0.647,0.785,0.4,0.0
|
| 40 |
+
5,F-Test,Non-search,Correlation,1229.0,1200.0,1257.0,22.8,0.679,0.846,0.0,0.0
|
| 41 |
+
6,Lasso,Non-search,Regularization,1226.0,1194.0,1255.0,20.9,0.362,0.73,2.1,0.0
|
| 42 |
+
7,MarkovBlanket,Search,Backward search,1220.0,1192.0,1247.0,21.2,0.413,0.463,0.1,0.0
|
| 43 |
+
8,(R)ReliefF,Non-search,Distance,1198.0,1166.0,1230.0,18.6,0.646,0.74,14.3,0.0
|
| 44 |
+
9,mRMR,Non-search,Information theory,1193.0,1165.0,1220.0,21.7,0.604,0.655,0.4,3.4
|
| 45 |
+
10,GainRatio,Non-search,Information theory,1183.0,1157.0,1211.0,19.5,0.644,0.825,0.2,0.0
|
| 46 |
+
11,SFS,Search,Forward search,1178.0,1146.0,1209.0,18.2,0.267,0.627,2105.8,43.5
|
| 47 |
+
12,SBE,Search,Backward search,1169.0,1130.0,1207.0,11.4,0.132,0.56,2495.5,46.5
|
| 48 |
+
13,LOCO,Search,Leave-one-out,1151.0,1119.0,1184.0,13.0,0.173,0.583,241.5,7.6
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,966.0,1032.0,0.0,0.001,0.5,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,965.0,913.0,1008.0,-18.6,0.652,0.797,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_all/budgets_all/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1310.0,1279.0,1342.0,6.15,63.2,15.1,,0.643,0.669,0.2,0.0,71,5325
|
| 3 |
-
2,CART,Non-search,Tree,1265.0,1236.0,1299.0,6.69,59.3,18.0,,0.422,0.586,0.1,0.0,71,5325
|
| 4 |
-
3,ElasticNet,Non-search,Regularization,1252.0,1219.0,1284.0,6.87,58.1,18.3,,0.447,0.76,7.6,0.0,71,5325
|
| 5 |
-
4,MI,Non-search,Information theory,1248.0,1224.0,1276.0,6.99,57.2,18.2,,0.647,0.785,0.4,0.0,71,5325
|
| 6 |
-
5,F-Test,Non-search,Correlation,1229.0,1200.0,1257.0,7.35,54.6,19.0,,0.679,0.846,0.0,0.0,71,5325
|
| 7 |
-
6,Lasso,Non-search,Regularization,1226.0,1194.0,1255.0,7.2,55.7,18.7,,0.362,0.73,2.1,0.0,71,5325
|
| 8 |
-
7,MarkovBlanket,Search,Backward search,1220.0,1192.0,1247.0,7.55,53.2,19.6,,0.413,0.463,0.1,0.0,71,5325
|
| 9 |
-
8,(R)ReliefF,Non-search,Distance,1198.0,1166.0,1230.0,8.0,50.0,20.6,,0.646,0.74,14.3,0.0,71,5325
|
| 10 |
-
9,mRMR,Non-search,Information theory,1193.0,1165.0,1220.0,7.96,50.3,19.9,,0.604,0.655,0.4,3.4,71,5325
|
| 11 |
-
10,GainRatio,Non-search,Information theory,1183.0,1157.0,1211.0,8.19,48.6,21.5,,0.644,0.825,0.2,0.0,71,5325
|
| 12 |
-
11,SFS,Search,Forward search,1178.0,1146.0,1209.0,7.99,50.1,21.7,,0.267,0.627,2105.8,43.5,71,5324
|
| 13 |
-
12,SBE,Search,Backward search,1169.0,1130.0,1207.0,8.31,47.8,23.7,,0.132,0.56,2495.5,46.5,71,5324
|
| 14 |
-
13,LOCO,Search,Leave-one-out,1151.0,1119.0,1184.0,8.39,47.2,23.8,,0.173,0.583,241.5,7.6,71,5325
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,966.0,1032.0,10.72,30.6,32.0,,0.001,0.5,0.0,0.0,71,5325
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,965.0,913.0,1008.0,11.64,24.0,36.6,,0.652,0.797,0.2,0.0,71,5325
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1310.0,1279.0,1342.0,6.15,63.2,15.1,26.2,,0.643,0.669,0.2,0.0,71,5325
|
| 3 |
+
2,CART,Non-search,Tree,1265.0,1236.0,1299.0,6.69,59.3,18.0,23.8,,0.422,0.586,0.1,0.0,71,5325
|
| 4 |
+
3,ElasticNet,Non-search,Regularization,1252.0,1219.0,1284.0,6.87,58.1,18.3,21.9,,0.447,0.76,7.6,0.0,71,5325
|
| 5 |
+
4,MI,Non-search,Information theory,1248.0,1224.0,1276.0,6.99,57.2,18.2,23.8,,0.647,0.785,0.4,0.0,71,5325
|
| 6 |
+
5,F-Test,Non-search,Correlation,1229.0,1200.0,1257.0,7.35,54.6,19.0,22.8,,0.679,0.846,0.0,0.0,71,5325
|
| 7 |
+
6,Lasso,Non-search,Regularization,1226.0,1194.0,1255.0,7.2,55.7,18.7,20.9,,0.362,0.73,2.1,0.0,71,5325
|
| 8 |
+
7,MarkovBlanket,Search,Backward search,1220.0,1192.0,1247.0,7.55,53.2,19.6,21.2,,0.413,0.463,0.1,0.0,71,5325
|
| 9 |
+
8,(R)ReliefF,Non-search,Distance,1198.0,1166.0,1230.0,8.0,50.0,20.6,18.6,,0.646,0.74,14.3,0.0,71,5325
|
| 10 |
+
9,mRMR,Non-search,Information theory,1193.0,1165.0,1220.0,7.96,50.3,19.9,21.7,,0.604,0.655,0.4,3.4,71,5325
|
| 11 |
+
10,GainRatio,Non-search,Information theory,1183.0,1157.0,1211.0,8.19,48.6,21.5,19.5,,0.644,0.825,0.2,0.0,71,5325
|
| 12 |
+
11,SFS,Search,Forward search,1178.0,1146.0,1209.0,7.99,50.1,21.7,18.2,,0.267,0.627,2105.8,43.5,71,5324
|
| 13 |
+
12,SBE,Search,Backward search,1169.0,1130.0,1207.0,8.31,47.8,23.7,11.4,,0.132,0.56,2495.5,46.5,71,5324
|
| 14 |
+
13,LOCO,Search,Leave-one-out,1151.0,1119.0,1184.0,8.39,47.2,23.8,13.0,,0.173,0.583,241.5,7.6,71,5325
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,966.0,1032.0,10.72,30.6,32.0,0.0,,0.001,0.5,0.0,0.0,71,5325
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,965.0,913.0,1008.0,11.64,24.0,36.6,-18.6,,0.652,0.797,0.2,0.0,71,5325
|
data/subsets/tasks_all/models_lightgbm/budgets_0/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1368.0'>1368<span class='sub'>+66 / −57</span></td><td class='num' data-sort='6.11'>6.11</td><td class='num' data-sort='18.1'>18.1</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1347.0'>1347<span class='sub'>+56 / −45</span></td><td class='num' data-sort='6.29'>6.29</td><td class='num' data-sort='20.1'>20.1</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+55 / −50</span></td><td class='num' data-sort='6.54'>6.54</td><td class='num' data-sort='21.1'>21.1</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1312.0'>1312<span class='sub'>+56 / −56</span></td><td class='num' data-sort='6.76'>6.76</td><td class='num' data-sort='21.5'>21.5</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='1052.3'>1,052.3</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1304.0'>1304<span class='sub'>+46 / −46</span></td><td class='num' data-sort='7.01'>7.01</td><td class='num' data-sort='22.5'>22.5</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1292.0'>1292<span class='sub'>+43 / −42</span></td><td class='num' data-sort='7.12'>7.12</td><td class='num' data-sort='22.0'>22.0</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1280.0'>1280<span class='sub'>+48 / −52</span></td><td class='num' data-sort='7.47'>7.47</td><td class='num' data-sort='22.8'>22.8</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1270.0'>1270<span class='sub'>+43 / −49</span></td><td class='num' data-sort='7.48'>7.48</td><td class='num' data-sort='24.0'>24.0</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1254.0'>1254<span class='sub'>+45 / −44</span></td><td class='num' data-sort='7.73'>7.73</td><td class='num' data-sort='22.7'>22.7</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1234.0'>1234<span class='sub'>+50 / −49</span></td><td class='num' data-sort='8.12'>8.12</td><td class='num' data-sort='25.1'>25.1</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1233.0'>1233<span class='sub'>+56 / −56</span></td><td class='num' data-sort='8.28'>8.28</td><td class='num' data-sort='27.0'>27.0</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3579.1'>3,579.1</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1194.0'>1194<span class='sub'>+47 / −47</span></td><td class='num' data-sort='8.42'>8.42</td><td class='num' data-sort='26.9'>26.9</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='240.9'>240.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1191.0'>1191<span class='sub'>+44 / −43</span></td><td class='num' data-sort='8.79'>8.79</td><td class='num' data-sort='28.4'>28.4</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+54 / −67</span></td><td class='num' data-sort='11.43'>11.43</td><td class='num' data-sort='39.7'>39.7</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='934.0'>934<span class='sub'>+75 / −97</span></td><td class='num' data-sort='12.45'>12.45</td><td class='num' data-sort='44.3'>44.3</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1368.0,1311.0,1434.0,
|
| 37 |
-
2,CART,Non-search,Tree,1347.0,1302.0,1403.0,
|
| 38 |
-
3,MarkovBlanket,Search,Backward search,1325.0,1275.0,1380.0,
|
| 39 |
-
4,SFS,Search,Forward search,1312.0,1256.0,1368.0,
|
| 40 |
-
5,ElasticNet,Non-search,Regularization,1304.0,1258.0,1350.0,
|
| 41 |
-
6,MI,Non-search,Information theory,1292.0,1250.0,1335.0,
|
| 42 |
-
7,F-Test,Non-search,Correlation,1280.0,1228.0,1328.0,
|
| 43 |
-
8,Lasso,Non-search,Regularization,1270.0,1221.0,1313.0,
|
| 44 |
-
9,mRMR,Non-search,Information theory,1254.0,1210.0,1299.0,
|
| 45 |
-
10,GainRatio,Non-search,Information theory,1234.0,1185.0,1284.0,
|
| 46 |
-
11,SBE,Search,Backward search,1233.0,1177.0,1289.0,
|
| 47 |
-
12,LOCO,Search,Leave-one-out,1194.0,1147.0,1241.0,
|
| 48 |
-
13,(R)ReliefF,Non-search,Distance,1191.0,1148.0,1235.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,933.0,1054.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,934.0,837.0,1009.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1368.0'>1368<span class='sub'>+66 / −57</span></td><td class='num' data-sort='29.5'>29.5</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1347.0'>1347<span class='sub'>+56 / −45</span></td><td class='num' data-sort='27.7'>27.7</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+55 / −50</span></td><td class='num' data-sort='25.8'>25.8</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1312.0'>1312<span class='sub'>+56 / −56</span></td><td class='num' data-sort='24.2'>24.2</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='1052.3'>1,052.3</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1304.0'>1304<span class='sub'>+46 / −46</span></td><td class='num' data-sort='25.2'>25.2</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1292.0'>1292<span class='sub'>+43 / −42</span></td><td class='num' data-sort='27.8'>27.8</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1280.0'>1280<span class='sub'>+48 / −52</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1270.0'>1270<span class='sub'>+43 / −49</span></td><td class='num' data-sort='23.8'>23.8</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1254.0'>1254<span class='sub'>+45 / −44</span></td><td class='num' data-sort='25.6'>25.6</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1234.0'>1234<span class='sub'>+50 / −49</span></td><td class='num' data-sort='23.3'>23.3</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1233.0'>1233<span class='sub'>+56 / −56</span></td><td class='num' data-sort='14.6'>14.6</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3579.1'>3,579.1</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1194.0'>1194<span class='sub'>+47 / −47</span></td><td class='num' data-sort='16.2'>16.2</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='240.9'>240.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1191.0'>1191<span class='sub'>+44 / −43</span></td><td class='num' data-sort='19.7'>19.7</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+54 / −67</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='934.0'>934<span class='sub'>+75 / −97</span></td><td class='num' data-sort='-19.0'>-19.0</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1368.0,1311.0,1434.0,29.5,0.628,0.805,0.2,0.0
|
| 37 |
+
2,CART,Non-search,Tree,1347.0,1302.0,1403.0,27.7,0.484,0.731,0.1,0.0
|
| 38 |
+
3,MarkovBlanket,Search,Backward search,1325.0,1275.0,1380.0,25.8,0.418,0.465,0.1,0.0
|
| 39 |
+
4,SFS,Search,Forward search,1312.0,1256.0,1368.0,24.2,0.368,0.75,1052.3,29.1
|
| 40 |
+
5,ElasticNet,Non-search,Regularization,1304.0,1258.0,1350.0,25.2,0.509,0.915,6.1,0.0
|
| 41 |
+
6,MI,Non-search,Information theory,1292.0,1250.0,1335.0,27.8,0.645,0.898,0.4,0.0
|
| 42 |
+
7,F-Test,Non-search,Correlation,1280.0,1228.0,1328.0,25.9,0.725,0.952,0.0,0.0
|
| 43 |
+
8,Lasso,Non-search,Regularization,1270.0,1221.0,1313.0,23.8,0.421,0.89,2.0,0.0
|
| 44 |
+
9,mRMR,Non-search,Information theory,1254.0,1210.0,1299.0,25.6,0.528,0.779,0.3,0.0
|
| 45 |
+
10,GainRatio,Non-search,Information theory,1234.0,1185.0,1284.0,23.3,0.679,0.942,0.2,0.0
|
| 46 |
+
11,SBE,Search,Backward search,1233.0,1177.0,1289.0,14.6,0.195,0.621,3579.1,49.3
|
| 47 |
+
12,LOCO,Search,Leave-one-out,1194.0,1147.0,1241.0,16.2,0.299,0.665,240.9,0.0
|
| 48 |
+
13,(R)ReliefF,Non-search,Distance,1191.0,1148.0,1235.0,19.7,0.642,0.873,13.2,0.0
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,933.0,1054.0,0.0,-0.001,0.493,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,934.0,837.0,1009.0,-19.0,0.588,0.847,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_lightgbm/budgets_0/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='157.8' x2='157.8' y1='16.2' y2='368.0'/><text class='sa-text' x='157.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='257.7' x2='257.7' y1='16.2' y2='368.0'/><text class='sa-text' x='257.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='357.7' x2='357.7' y1='16.2' y2='368.0'/><text class='sa-text' x='357.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='457.7' x2='457.7' y1='16.2' y2='368.0'/><text class='sa-text' x='457.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='557.6' x2='557.6' y1='16.2' y2='368.0'/><text class='sa-text' x='557.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.6' x2='657.6' y1='16.2' y2='368.0'/><text class='sa-text' x='657.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.6' x2='757.6' y1='16.2' y2='368.0'/><text class='sa-text' x='757.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='297.9' y2='297.9'/><text class='sa-text' x='58.0' y='303.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='243.8' y2='243.8'/><text class='sa-text' x='58.0' y='249.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='189.6' y2='189.6'/><text class='sa-text' x='58.0' y='195.6' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='135.5' y2='135.5'/><text class='sa-text' x='58.0' y='141.5' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='81.4' y2='81.4'/><text class='sa-text' x='58.0' y='87.4' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='27.2' y2='27.2'/><text class='sa-text' x='58.0' y='33.2' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 338.9,71.3 473.8,52.4 516.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='523.9' cy='50.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='473.8' cy='52.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='338.9' cy='71.3' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='485.4' cy='73.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='478.9' cy='74.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='690.7' cy='79.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='783.4' cy='90.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.7' cy='93.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.1' cy='100.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.4' cy='138.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.7' cy='176.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='193.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.5' y1='32.2' x2='637.4' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='637.4' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='523.9' y1='50.9' x2='588.8' y2='57.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='588.8' y='57.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='473.8' y1='52.4' x2='466.9' y2='26.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='466.9' y='26.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='338.9' y1='71.3' x2='304.9' y2='91.3' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='304.9' y='91.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='485.4' y1='73.0' x2='588.7' y2='86.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='588.7' y='86.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='478.9' y1='74.8' x2='418.4' y2='53.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='418.4' y='53.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='690.7' y1='79.5' x2='777.5' y2='64.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='777.5' y='64.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='783.4' y1='90.3' x2='812.4' y2='102.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='812.4' y='102.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='686.7' y1='93.8' x2='734.2' y2='120.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='734.2' y='120.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='468.1' y1='100.2' x2='459.3' y2='125.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='459.3' y='125.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='797.0' y='154.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='737.7' y1='176.5' x2='704.9' y2='190.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='704.9' y='190.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='207.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 29.5% Selection time per 1k features: 38.8 s Datasets: 71 Next to MI: 27.8%, 46 s' aria-label='RFImportance (on the Pareto front) Improvement over Random: 29.5% Selection time per 1k features: 38.8 s Datasets: 71 Next to MI: 27.8%, 46 s'/><circle class='sa-hit' cx='523.9' cy='50.9' r='10.5' data-tip='MI Improvement over Random: 27.8% Selection time per 1k features: 46 s Datasets: 71 Next to RFImportance: 29.5%, 38.8 s' aria-label='MI Improvement over Random: 27.8% Selection time per 1k features: 46 s Datasets: 71 Next to RFImportance: 29.5%, 38.8 s'/><circle class='sa-hit' cx='473.8' cy='52.4' r='10.5' data-tip='CART (on the Pareto front) Improvement over Random: 27.7% Selection time per 1k features: 14.5 s Datasets: 71' aria-label='CART (on the Pareto front) Improvement over Random: 27.7% Selection time per 1k features: 14.5 s Datasets: 71'/><circle class='sa-hit' cx='338.9' cy='71.3' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 25.9% Selection time per 1k features: 0.648 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 25.9% Selection time per 1k features: 0.648 s Datasets: 71'/><circle class='sa-hit' cx='485.4' cy='73.0' r='10.5' data-tip='MarkovBlanket Improvement over Random: 25.8% Selection time per 1k features: 19 s Datasets: 71 Next to mRMR: 25.6%, 16.3 s' aria-label='MarkovBlanket Improvement over Random: 25.8% Selection time per 1k features: 19 s Datasets: 71 Next to mRMR: 25.6%, 16.3 s'/><circle class='sa-hit' cx='478.9' cy='74.8' r='10.5' data-tip='mRMR Improvement over Random: 25.6% Selection time per 1k features: 16.3 s Datasets: 71 Next to MarkovBlanket: 25.8%, 19 s' aria-label='mRMR Improvement over Random: 25.6% Selection time per 1k features: 16.3 s Datasets: 71 Next to MarkovBlanket: 25.8%, 19 s'/><circle class='sa-hit' cx='690.7' cy='79.5' r='10.5' data-tip='ElasticNet Improvement over Random: 25.2% Selection time per 1k features: 2.14e+03 s Datasets: 71 Next to Lasso: 23.8%, 1.96e+03 s' aria-label='ElasticNet Improvement over Random: 25.2% Selection time per 1k features: 2.14e+03 s Datasets: 71 Next to Lasso: 23.8%, 1.96e+03 s'/><circle class='sa-hit' cx='783.4' cy='90.3' r='10.5' data-tip='SFS Improvement over Random: 24.2% Selection time per 1k features: 1.81e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 24.2% Selection time per 1k features: 1.81e+04 s Datasets: 71'/><circle class='sa-hit' cx='686.7' cy='93.8' r='10.5' data-tip='Lasso Improvement over Random: 23.8% Selection time per 1k features: 1.96e+03 s Datasets: 71 Next to ElasticNet: 25.2%, 2.14e+03 s' aria-label='Lasso Improvement over Random: 23.8% Selection time per 1k features: 1.96e+03 s Datasets: 71 Next to ElasticNet: 25.2%, 2.14e+03 s'/><circle class='sa-hit' cx='468.1' cy='100.2' r='10.5' data-tip='GainRatio Improvement over Random: 23.3% Selection time per 1k features: 12.7 s Datasets: 71' aria-label='GainRatio Improvement over Random: 23.3% Selection time per 1k features: 12.7 s Datasets: 71'/><circle class='sa-hit' cx='795.4' cy='138.5' r='10.5' data-tip='(R)ReliefF Improvement over Random: 19.7% Selection time per 1k features: 2.39e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 19.7% Selection time per 1k features: 2.39e+04 s Datasets: 71'/><circle class='sa-hit' cx='737.7' cy='176.5' r='10.5' data-tip='LOCO Improvement over Random: 16.2% Selection time per 1k features: 6.32e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 16.2% Selection time per 1k features: 6.32e+03 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='193.5' r='10.5' data-tip='SBE Improvement over Random: 14.6% Selection time per 1k features: 3.53e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 14.6% Selection time per 1k features: 3.53e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0028 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0028 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,29.5432,38.7674,71,True
|
| 37 |
MI,27.8197,45.9925,71,False
|
| 38 |
CART,27.6757,14.4968,71,True
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='157.8' x2='157.8' y1='16.2' y2='368.0'/><text class='sa-text' x='157.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='257.7' x2='257.7' y1='16.2' y2='368.0'/><text class='sa-text' x='257.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='357.7' x2='357.7' y1='16.2' y2='368.0'/><text class='sa-text' x='357.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='457.7' x2='457.7' y1='16.2' y2='368.0'/><text class='sa-text' x='457.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='557.6' x2='557.6' y1='16.2' y2='368.0'/><text class='sa-text' x='557.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.6' x2='657.6' y1='16.2' y2='368.0'/><text class='sa-text' x='657.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.6' x2='757.6' y1='16.2' y2='368.0'/><text class='sa-text' x='757.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='58.0' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='297.9' y2='297.9'/><text class='sa-text' x='58.0' y='303.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='243.8' y2='243.8'/><text class='sa-text' x='58.0' y='249.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='189.6' y2='189.6'/><text class='sa-text' x='58.0' y='195.6' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='135.5' y2='135.5'/><text class='sa-text' x='58.0' y='141.5' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='81.4' y2='81.4'/><text class='sa-text' x='58.0' y='87.4' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='27.2' y2='27.2'/><text class='sa-text' x='58.0' y='33.2' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='16.2' width='780.8' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='51.9' x2='69.0' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.0 338.9,71.3 473.8,52.4 516.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='523.9' cy='50.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='473.8' cy='52.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='338.9' cy='71.3' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='485.4' cy='73.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='478.9' cy='74.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='690.7' cy='79.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='783.4' cy='90.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='686.7' cy='93.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.1' cy='100.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.4' cy='138.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.7' cy='176.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='193.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.5' y1='32.2' x2='646.8' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='646.8' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='523.9' y1='50.9' x2='581.9' y2='53.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='581.9' y='53.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='473.8' y1='52.4' x2='468.1' y2='26.2' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='468.1' y='26.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='338.9' y1='71.3' x2='309.3' y2='92.4' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='309.3' y='92.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='485.4' y1='73.0' x2='578.0' y2='91.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='578.0' y='91.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='478.9' y1='74.8' x2='419.7' y2='54.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='419.7' y='54.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='690.7' y1='79.5' x2='785.7' y2='59.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='785.7' y='59.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='783.4' y1='90.3' x2='812.4' y2='102.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='812.4' y='102.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='686.7' y1='93.8' x2='735.5' y2='120.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='735.5' y='120.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><text class='sa-dim sa-halo' x='465.0' y='120.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='797.0' y='152.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='737.7' y1='176.5' x2='704.9' y2='190.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='704.9' y='190.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='207.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 29.5% Selection time per 1k features: 38.8 s Datasets: 71 Next to MI: 27.8%, 46 s' aria-label='RFImportance (on the Pareto front) Improvement over Random: 29.5% Selection time per 1k features: 38.8 s Datasets: 71 Next to MI: 27.8%, 46 s'/><circle class='sa-hit' cx='523.9' cy='50.9' r='10.5' data-tip='MI Improvement over Random: 27.8% Selection time per 1k features: 46 s Datasets: 71 Next to RFImportance: 29.5%, 38.8 s' aria-label='MI Improvement over Random: 27.8% Selection time per 1k features: 46 s Datasets: 71 Next to RFImportance: 29.5%, 38.8 s'/><circle class='sa-hit' cx='473.8' cy='52.4' r='10.5' data-tip='CART (on the Pareto front) Improvement over Random: 27.7% Selection time per 1k features: 14.5 s Datasets: 71' aria-label='CART (on the Pareto front) Improvement over Random: 27.7% Selection time per 1k features: 14.5 s Datasets: 71'/><circle class='sa-hit' cx='338.9' cy='71.3' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 25.9% Selection time per 1k features: 0.648 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 25.9% Selection time per 1k features: 0.648 s Datasets: 71'/><circle class='sa-hit' cx='485.4' cy='73.0' r='10.5' data-tip='MarkovBlanket Improvement over Random: 25.8% Selection time per 1k features: 19 s Datasets: 71 Next to mRMR: 25.6%, 16.3 s' aria-label='MarkovBlanket Improvement over Random: 25.8% Selection time per 1k features: 19 s Datasets: 71 Next to mRMR: 25.6%, 16.3 s'/><circle class='sa-hit' cx='478.9' cy='74.8' r='10.5' data-tip='mRMR Improvement over Random: 25.6% Selection time per 1k features: 16.3 s Datasets: 71 Next to MarkovBlanket: 25.8%, 19 s' aria-label='mRMR Improvement over Random: 25.6% Selection time per 1k features: 16.3 s Datasets: 71 Next to MarkovBlanket: 25.8%, 19 s'/><circle class='sa-hit' cx='690.7' cy='79.5' r='10.5' data-tip='ElasticNet Improvement over Random: 25.2% Selection time per 1k features: 2.14e+03 s Datasets: 71 Next to Lasso: 23.8%, 1.96e+03 s' aria-label='ElasticNet Improvement over Random: 25.2% Selection time per 1k features: 2.14e+03 s Datasets: 71 Next to Lasso: 23.8%, 1.96e+03 s'/><circle class='sa-hit' cx='783.4' cy='90.3' r='10.5' data-tip='SFS Improvement over Random: 24.2% Selection time per 1k features: 1.81e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 24.2% Selection time per 1k features: 1.81e+04 s Datasets: 71'/><circle class='sa-hit' cx='686.7' cy='93.8' r='10.5' data-tip='Lasso Improvement over Random: 23.8% Selection time per 1k features: 1.96e+03 s Datasets: 71 Next to ElasticNet: 25.2%, 2.14e+03 s' aria-label='Lasso Improvement over Random: 23.8% Selection time per 1k features: 1.96e+03 s Datasets: 71 Next to ElasticNet: 25.2%, 2.14e+03 s'/><circle class='sa-hit' cx='468.1' cy='100.2' r='10.5' data-tip='GainRatio Improvement over Random: 23.3% Selection time per 1k features: 12.7 s Datasets: 71' aria-label='GainRatio Improvement over Random: 23.3% Selection time per 1k features: 12.7 s Datasets: 71'/><circle class='sa-hit' cx='795.4' cy='138.5' r='10.5' data-tip='(R)ReliefF Improvement over Random: 19.7% Selection time per 1k features: 2.39e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 19.7% Selection time per 1k features: 2.39e+04 s Datasets: 71'/><circle class='sa-hit' cx='737.7' cy='176.5' r='10.5' data-tip='LOCO Improvement over Random: 16.2% Selection time per 1k features: 6.32e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 16.2% Selection time per 1k features: 6.32e+03 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='193.5' r='10.5' data-tip='SBE Improvement over Random: 14.6% Selection time per 1k features: 3.53e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 14.6% Selection time per 1k features: 3.53e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0028 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0028 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,29.5432,38.7674,71,True
|
| 37 |
MI,27.8197,45.9925,71,False
|
| 38 |
CART,27.6757,14.4968,71,True
|
data/subsets/tasks_all/models_lightgbm/budgets_0/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1368.0,1311.0,1434.0,6.11,63.5,18.1,,0.628,0.805,0.2,0.0,71,213
|
| 3 |
-
2,CART,Non-search,Tree,1347.0,1302.0,1403.0,6.29,62.2,20.1,,0.484,0.731,0.1,0.0,71,213
|
| 4 |
-
3,MarkovBlanket,Search,Backward search,1325.0,1275.0,1380.0,6.54,60.4,21.1,,0.418,0.465,0.1,0.0,71,213
|
| 5 |
-
4,SFS,Search,Forward search,1312.0,1256.0,1368.0,6.76,58.8,21.5,,0.368,0.75,1052.3,29.1,71,213
|
| 6 |
-
5,ElasticNet,Non-search,Regularization,1304.0,1258.0,1350.0,7.01,57.1,22.5,,0.509,0.915,6.1,0.0,71,213
|
| 7 |
-
6,MI,Non-search,Information theory,1292.0,1250.0,1335.0,7.12,56.3,22.0,,0.645,0.898,0.4,0.0,71,213
|
| 8 |
-
7,F-Test,Non-search,Correlation,1280.0,1228.0,1328.0,7.47,53.8,22.8,,0.725,0.952,0.0,0.0,71,213
|
| 9 |
-
8,Lasso,Non-search,Regularization,1270.0,1221.0,1313.0,7.48,53.7,24.0,,0.421,0.89,2.0,0.0,71,213
|
| 10 |
-
9,mRMR,Non-search,Information theory,1254.0,1210.0,1299.0,7.73,52.0,22.7,,0.528,0.779,0.3,0.0,71,213
|
| 11 |
-
10,GainRatio,Non-search,Information theory,1234.0,1185.0,1284.0,8.12,49.2,25.1,,0.679,0.942,0.2,0.0,71,213
|
| 12 |
-
11,SBE,Search,Backward search,1233.0,1177.0,1289.0,8.28,48.0,27.0,,0.195,0.621,3579.1,49.3,71,213
|
| 13 |
-
12,LOCO,Search,Leave-one-out,1194.0,1147.0,1241.0,8.42,47.0,26.9,,0.299,0.665,240.9,0.0,71,213
|
| 14 |
-
13,(R)ReliefF,Non-search,Distance,1191.0,1148.0,1235.0,8.79,44.4,28.4,,0.642,0.873,13.2,0.0,71,213
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,933.0,1054.0,11.43,25.5,39.7,,-0.001,0.493,0.0,0.0,71,213
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,934.0,837.0,1009.0,12.45,18.2,44.3,,0.588,0.847,0.2,0.0,71,213
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1368.0,1311.0,1434.0,6.11,63.5,18.1,29.5,,0.628,0.805,0.2,0.0,71,213
|
| 3 |
+
2,CART,Non-search,Tree,1347.0,1302.0,1403.0,6.29,62.2,20.1,27.7,,0.484,0.731,0.1,0.0,71,213
|
| 4 |
+
3,MarkovBlanket,Search,Backward search,1325.0,1275.0,1380.0,6.54,60.4,21.1,25.8,,0.418,0.465,0.1,0.0,71,213
|
| 5 |
+
4,SFS,Search,Forward search,1312.0,1256.0,1368.0,6.76,58.8,21.5,24.2,,0.368,0.75,1052.3,29.1,71,213
|
| 6 |
+
5,ElasticNet,Non-search,Regularization,1304.0,1258.0,1350.0,7.01,57.1,22.5,25.2,,0.509,0.915,6.1,0.0,71,213
|
| 7 |
+
6,MI,Non-search,Information theory,1292.0,1250.0,1335.0,7.12,56.3,22.0,27.8,,0.645,0.898,0.4,0.0,71,213
|
| 8 |
+
7,F-Test,Non-search,Correlation,1280.0,1228.0,1328.0,7.47,53.8,22.8,25.9,,0.725,0.952,0.0,0.0,71,213
|
| 9 |
+
8,Lasso,Non-search,Regularization,1270.0,1221.0,1313.0,7.48,53.7,24.0,23.8,,0.421,0.89,2.0,0.0,71,213
|
| 10 |
+
9,mRMR,Non-search,Information theory,1254.0,1210.0,1299.0,7.73,52.0,22.7,25.6,,0.528,0.779,0.3,0.0,71,213
|
| 11 |
+
10,GainRatio,Non-search,Information theory,1234.0,1185.0,1284.0,8.12,49.2,25.1,23.3,,0.679,0.942,0.2,0.0,71,213
|
| 12 |
+
11,SBE,Search,Backward search,1233.0,1177.0,1289.0,8.28,48.0,27.0,14.6,,0.195,0.621,3579.1,49.3,71,213
|
| 13 |
+
12,LOCO,Search,Leave-one-out,1194.0,1147.0,1241.0,8.42,47.0,26.9,16.2,,0.299,0.665,240.9,0.0,71,213
|
| 14 |
+
13,(R)ReliefF,Non-search,Distance,1191.0,1148.0,1235.0,8.79,44.4,28.4,19.7,,0.642,0.873,13.2,0.0,71,213
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,933.0,1054.0,11.43,25.5,39.7,0.0,,-0.001,0.493,0.0,0.0,71,213
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,934.0,837.0,1009.0,12.45,18.2,44.3,-19.0,,0.588,0.847,0.2,0.0,71,213
|
data/subsets/tasks_all/models_lightgbm/budgets_1/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+57 / −52</span></td><td class='num' data-sort='6.45'>6.45</td><td class='num' data-sort='16.8'>16.8</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+49 / −46</span></td><td class='num' data-sort='6.43'>6.43</td><td class='num' data-sort='18.4'>18.4</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1355.0'>1355<span class='sub'>+49 / −48</span></td><td class='num' data-sort='6.79'>6.79</td><td class='num' data-sort='19.1'>19.1</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1340.0'>1340<span class='sub'>+47 / −40</span></td><td class='num' data-sort='7.03'>7.03</td><td class='num' data-sort='20.8'>20.8</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1338.0'>1338<span class='sub'>+48 / −50</span></td><td class='num' data-sort='6.73'>6.73</td><td class='num' data-sort='18.4'>18.4</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.1'>2.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+41 / −36</span></td><td class='num' data-sort='7.01'>7.01</td><td class='num' data-sort='19.5'>19.5</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1311.0'>1311<span class='sub'>+50 / −48</span></td><td class='num' data-sort='7.63'>7.63</td><td class='num' data-sort='21.1'>21.1</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1294.0'>1294<span class='sub'>+51 / −50</span></td><td class='num' data-sort='7.71'>7.71</td><td class='num' data-sort='22.7'>22.7</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1770.9'>1,770.9</td><td class='num' data-sort='44.1'>44.1</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1291.0'>1291<span class='sub'>+47 / −44</span></td><td class='num' data-sort='7.77'>7.77</td><td class='num' data-sort='22.6'>22.6</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1289.0'>1289<span class='sub'>+47 / −48</span></td><td class='num' data-sort='7.73'>7.73</td><td class='num' data-sort='22.0'>22.0</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1275.0'>1275<span class='sub'>+49 / −50</span></td><td class='num' data-sort='8.04'>8.04</td><td class='num' data-sort='22.5'>22.5</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='12.8'>12.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1259.0'>1259<span class='sub'>+55 / −59</span></td><td class='num' data-sort='8.01'>8.01</td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3152.6'>3,152.6</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1241.0'>1241<span class='sub'>+55 / −51</span></td><td class='num' data-sort='8.54'>8.54</td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='239.5'>239.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+44 / −53</span></td><td class='num' data-sort='11.82'>11.82</td><td class='num' data-sort='38.0'>38.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='995.0'>995<span class='sub'>+67 / −88</span></td><td class='num' data-sort='12.32'>12.32</td><td class='num' data-sort='41.7'>41.7</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1366.0,1314.0,1423.0,
|
| 37 |
-
2,ElasticNet,Non-search,Regularization,1366.0,1320.0,1415.0,
|
| 38 |
-
3,CART,Non-search,Tree,1355.0,1307.0,1404.0,
|
| 39 |
-
4,MarkovBlanket,Search,Backward search,1340.0,1300.0,1387.0,
|
| 40 |
-
5,Lasso,Non-search,Regularization,1338.0,1288.0,1386.0,
|
| 41 |
-
6,MI,Non-search,Information theory,1325.0,1289.0,1366.0,
|
| 42 |
-
7,mRMR,Non-search,Information theory,1311.0,1263.0,1361.0,
|
| 43 |
-
8,SFS,Search,Forward search,1294.0,1244.0,1345.0,
|
| 44 |
-
9,GainRatio,Non-search,Information theory,1291.0,1247.0,1338.0,
|
| 45 |
-
10,F-Test,Non-search,Correlation,1289.0,1241.0,1336.0,
|
| 46 |
-
11,(R)ReliefF,Non-search,Distance,1275.0,1225.0,1324.0,
|
| 47 |
-
12,SBE,Search,Backward search,1259.0,1200.0,1314.0,
|
| 48 |
-
13,LOCO,Search,Leave-one-out,1241.0,1190.0,1296.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,947.0,1044.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,995.0,907.0,1062.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+57 / −52</span></td><td class='num' data-sort='28.1'>28.1</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1366.0'>1366<span class='sub'>+49 / −46</span></td><td class='num' data-sort='25.7'>25.7</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1355.0'>1355<span class='sub'>+49 / −48</span></td><td class='num' data-sort='26.5'>26.5</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1340.0'>1340<span class='sub'>+47 / −40</span></td><td class='num' data-sort='24.8'>24.8</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1338.0'>1338<span class='sub'>+48 / −50</span></td><td class='num' data-sort='25.4'>25.4</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.1'>2.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1325.0'>1325<span class='sub'>+41 / −36</span></td><td class='num' data-sort='26.8'>26.8</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1311.0'>1311<span class='sub'>+50 / −48</span></td><td class='num' data-sort='25.0'>25.0</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1294.0'>1294<span class='sub'>+51 / −50</span></td><td class='num' data-sort='21.3'>21.3</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1770.9'>1,770.9</td><td class='num' data-sort='44.1'>44.1</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1291.0'>1291<span class='sub'>+47 / −44</span></td><td class='num' data-sort='24.3'>24.3</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1289.0'>1289<span class='sub'>+47 / −48</span></td><td class='num' data-sort='24.5'>24.5</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1275.0'>1275<span class='sub'>+49 / −50</span></td><td class='num' data-sort='21.9'>21.9</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='12.8'>12.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1259.0'>1259<span class='sub'>+55 / −59</span></td><td class='num' data-sort='15.5'>15.5</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3152.6'>3,152.6</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1241.0'>1241<span class='sub'>+55 / −51</span></td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='239.5'>239.5</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+44 / −53</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='995.0'>995<span class='sub'>+67 / −88</span></td><td class='num' data-sort='-29.1'>-29.1</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1366.0,1314.0,1423.0,28.1,0.653,0.741,0.2,0.0
|
| 37 |
+
2,ElasticNet,Non-search,Regularization,1366.0,1320.0,1415.0,25.7,0.519,0.873,6.3,0.0
|
| 38 |
+
3,CART,Non-search,Tree,1355.0,1307.0,1404.0,26.5,0.456,0.638,0.0,0.0
|
| 39 |
+
4,MarkovBlanket,Search,Backward search,1340.0,1300.0,1387.0,24.8,0.396,0.467,0.1,0.0
|
| 40 |
+
5,Lasso,Non-search,Regularization,1338.0,1288.0,1386.0,25.4,0.422,0.825,2.1,0.0
|
| 41 |
+
6,MI,Non-search,Information theory,1325.0,1289.0,1366.0,26.8,0.673,0.863,0.4,0.0
|
| 42 |
+
7,mRMR,Non-search,Information theory,1311.0,1263.0,1361.0,25.0,0.578,0.712,0.3,0.0
|
| 43 |
+
8,SFS,Search,Forward search,1294.0,1244.0,1345.0,21.3,0.278,0.664,1770.9,44.1
|
| 44 |
+
9,GainRatio,Non-search,Information theory,1291.0,1247.0,1338.0,24.3,0.681,0.915,0.1,0.0
|
| 45 |
+
10,F-Test,Non-search,Correlation,1289.0,1241.0,1336.0,24.5,0.734,0.932,0.0,0.0
|
| 46 |
+
11,(R)ReliefF,Non-search,Distance,1275.0,1225.0,1324.0,21.9,0.647,0.808,12.8,0.0
|
| 47 |
+
12,SBE,Search,Backward search,1259.0,1200.0,1314.0,15.5,0.175,0.578,3152.6,48.8
|
| 48 |
+
13,LOCO,Search,Leave-one-out,1241.0,1190.0,1296.0,15.9,0.22,0.613,239.5,0.0
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,947.0,1044.0,0.0,0.001,0.499,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,995.0,907.0,1062.0,-29.1,0.644,0.85,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_lightgbm/budgets_1/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1366.0,1314.0,1423.0,6.45,61.1,16.8,,0.653,0.741,0.2,0.0,71,213
|
| 3 |
-
2,ElasticNet,Non-search,Regularization,1366.0,1320.0,1415.0,6.43,61.2,18.4,,0.519,0.873,6.3,0.0,71,213
|
| 4 |
-
3,CART,Non-search,Tree,1355.0,1307.0,1404.0,6.79,58.6,19.1,,0.456,0.638,0.0,0.0,71,213
|
| 5 |
-
4,MarkovBlanket,Search,Backward search,1340.0,1300.0,1387.0,7.03,56.9,20.8,,0.396,0.467,0.1,0.0,71,213
|
| 6 |
-
5,Lasso,Non-search,Regularization,1338.0,1288.0,1386.0,6.73,59.1,18.4,,0.422,0.825,2.1,0.0,71,213
|
| 7 |
-
6,MI,Non-search,Information theory,1325.0,1289.0,1366.0,7.01,57.1,19.5,,0.673,0.863,0.4,0.0,71,213
|
| 8 |
-
7,mRMR,Non-search,Information theory,1311.0,1263.0,1361.0,7.63,52.6,21.1,,0.578,0.712,0.3,0.0,71,213
|
| 9 |
-
8,SFS,Search,Forward search,1294.0,1244.0,1345.0,7.71,52.1,22.7,,0.278,0.664,1770.9,44.1,71,213
|
| 10 |
-
9,GainRatio,Non-search,Information theory,1291.0,1247.0,1338.0,7.77,51.7,22.6,,0.681,0.915,0.1,0.0,71,213
|
| 11 |
-
10,F-Test,Non-search,Correlation,1289.0,1241.0,1336.0,7.73,51.9,22.0,,0.734,0.932,0.0,0.0,71,213
|
| 12 |
-
11,(R)ReliefF,Non-search,Distance,1275.0,1225.0,1324.0,8.04,49.7,22.5,,0.647,0.808,12.8,0.0,71,213
|
| 13 |
-
12,SBE,Search,Backward search,1259.0,1200.0,1314.0,8.01,50.0,25.5,,0.175,0.578,3152.6,48.8,71,213
|
| 14 |
-
13,LOCO,Search,Leave-one-out,1241.0,1190.0,1296.0,8.54,46.1,25.5,,0.22,0.613,239.5,0.0,71,213
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,947.0,1044.0,11.82,22.7,38.0,,0.001,0.499,0.0,0.0,71,213
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,995.0,907.0,1062.0,12.32,19.1,41.7,,0.644,0.85,0.2,0.0,71,213
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1366.0,1314.0,1423.0,6.45,61.1,16.8,28.1,,0.653,0.741,0.2,0.0,71,213
|
| 3 |
+
2,ElasticNet,Non-search,Regularization,1366.0,1320.0,1415.0,6.43,61.2,18.4,25.7,,0.519,0.873,6.3,0.0,71,213
|
| 4 |
+
3,CART,Non-search,Tree,1355.0,1307.0,1404.0,6.79,58.6,19.1,26.5,,0.456,0.638,0.0,0.0,71,213
|
| 5 |
+
4,MarkovBlanket,Search,Backward search,1340.0,1300.0,1387.0,7.03,56.9,20.8,24.8,,0.396,0.467,0.1,0.0,71,213
|
| 6 |
+
5,Lasso,Non-search,Regularization,1338.0,1288.0,1386.0,6.73,59.1,18.4,25.4,,0.422,0.825,2.1,0.0,71,213
|
| 7 |
+
6,MI,Non-search,Information theory,1325.0,1289.0,1366.0,7.01,57.1,19.5,26.8,,0.673,0.863,0.4,0.0,71,213
|
| 8 |
+
7,mRMR,Non-search,Information theory,1311.0,1263.0,1361.0,7.63,52.6,21.1,25.0,,0.578,0.712,0.3,0.0,71,213
|
| 9 |
+
8,SFS,Search,Forward search,1294.0,1244.0,1345.0,7.71,52.1,22.7,21.3,,0.278,0.664,1770.9,44.1,71,213
|
| 10 |
+
9,GainRatio,Non-search,Information theory,1291.0,1247.0,1338.0,7.77,51.7,22.6,24.3,,0.681,0.915,0.1,0.0,71,213
|
| 11 |
+
10,F-Test,Non-search,Correlation,1289.0,1241.0,1336.0,7.73,51.9,22.0,24.5,,0.734,0.932,0.0,0.0,71,213
|
| 12 |
+
11,(R)ReliefF,Non-search,Distance,1275.0,1225.0,1324.0,8.04,49.7,22.5,21.9,,0.647,0.808,12.8,0.0,71,213
|
| 13 |
+
12,SBE,Search,Backward search,1259.0,1200.0,1314.0,8.01,50.0,25.5,15.5,,0.175,0.578,3152.6,48.8,71,213
|
| 14 |
+
13,LOCO,Search,Leave-one-out,1241.0,1190.0,1296.0,8.54,46.1,25.5,15.9,,0.22,0.613,239.5,0.0,71,213
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,947.0,1044.0,11.82,22.7,38.0,0.0,,0.001,0.499,0.0,0.0,71,213
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,995.0,907.0,1062.0,12.32,19.1,41.7,-29.1,,0.644,0.85,0.2,0.0,71,213
|
data/subsets/tasks_all/models_lightgbm/budgets_2/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1321.0'>1321<span class='sub'>+59 / −50</span></td><td class='num' data-sort='6.15'>6.15</td><td class='num' data-sort='12.2'>12.2</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1283.0'>1283<span class='sub'>+48 / −47</span></td><td class='num' data-sort='6.72'>6.72</td><td class='num' data-sort='16.5'>16.5</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1282.0'>1282<span class='sub'>+49 / −50</span></td><td class='num' data-sort='6.86'>6.86</td><td class='num' data-sort='15.7'>15.7</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1268.0'>1268<span class='sub'>+51 / −51</span></td><td class='num' data-sort='6.95'>6.95</td><td class='num' data-sort='15.4'>15.4</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+49 / −49</span></td><td class='num' data-sort='7.01'>7.01</td><td class='num' data-sort='16.3'>16.3</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1248.0'>1248<span class='sub'>+38 / −38</span></td><td class='num' data-sort='7.3'>7.30</td><td class='num' data-sort='17.8'>17.8</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1224.0'>1224<span class='sub'>+58 / −53</span></td><td class='num' data-sort='7.76'>7.76</td><td class='num' data-sort='21.9'>21.9</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2887.2'>2,887.2</td><td class='num' data-sort='47.4'>47.4</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1212.0'>1212<span class='sub'>+47 / −45</span></td><td class='num' data-sort='7.66'>7.66</td><td class='num' data-sort='18.3'>18.3</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1205.0'>1205<span class='sub'>+47 / −45</span></td><td class='num' data-sort='8.25'>8.25</td><td class='num' data-sort='19.7'>19.7</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1199.0'>1199<span class='sub'>+44 / −47</span></td><td class='num' data-sort='7.77'>7.77</td><td class='num' data-sort='17.9'>17.9</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='12.9'>12.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+43 / −44</span></td><td class='num' data-sort='8.29'>8.29</td><td class='num' data-sort='19.2'>19.2</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1193.0'>1193<span class='sub'>+51 / −52</span></td><td class='num' data-sort='8.11'>8.11</td><td class='num' data-sort='20.8'>20.8</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2274.8'>2,274.8</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1175.0'>1175<span class='sub'>+55 / −56</span></td><td class='num' data-sort='8.2'>8.20</td><td class='num' data-sort='22.8'>22.8</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='251.9'>251.9</td><td class='num' data-sort='6.6'>6.6</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+46 / −52</span></td><td class='num' data-sort='10.87'>10.87</td><td class='num' data-sort='32.0'>32.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='939.0'>939<span class='sub'>+74 / −91</span></td><td class='num' data-sort='12.1'>12.10</td><td class='num' data-sort='37.6'>37.6</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1321.0,1271.0,1380.0,
|
| 37 |
-
2,CART,Non-search,Tree,1283.0,1236.0,1331.0,
|
| 38 |
-
3,MI,Non-search,Information theory,1282.0,1232.0,1331.0,
|
| 39 |
-
4,Lasso,Non-search,Regularization,1268.0,1217.0,1319.0,
|
| 40 |
-
5,ElasticNet,Non-search,Regularization,1252.0,1203.0,1301.0,
|
| 41 |
-
6,MarkovBlanket,Search,Backward search,1248.0,1210.0,1286.0,
|
| 42 |
-
7,SBE,Search,Backward search,1224.0,1171.0,1282.0,
|
| 43 |
-
8,F-Test,Non-search,Correlation,1212.0,1167.0,1259.0,
|
| 44 |
-
9,GainRatio,Non-search,Information theory,1205.0,1160.0,1252.0,
|
| 45 |
-
10,(R)ReliefF,Non-search,Distance,1199.0,1152.0,1243.0,
|
| 46 |
-
11,mRMR,Non-search,Information theory,1197.0,1153.0,1240.0,
|
| 47 |
-
12,SFS,Search,Forward search,1193.0,1141.0,1244.0,
|
| 48 |
-
13,LOCO,Search,Leave-one-out,1175.0,1119.0,1230.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,948.0,1046.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,939.0,848.0,1013.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1321.0'>1321<span class='sub'>+59 / −50</span></td><td class='num' data-sort='22.8'>22.8</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1283.0'>1283<span class='sub'>+48 / −47</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1282.0'>1282<span class='sub'>+49 / −50</span></td><td class='num' data-sort='19.5'>19.5</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1268.0'>1268<span class='sub'>+51 / −51</span></td><td class='num' data-sort='18.7'>18.7</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+49 / −49</span></td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1248.0'>1248<span class='sub'>+38 / −38</span></td><td class='num' data-sort='17.8'>17.8</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1224.0'>1224<span class='sub'>+58 / −53</span></td><td class='num' data-sort='11.4'>11.4</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2887.2'>2,887.2</td><td class='num' data-sort='47.4'>47.4</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1212.0'>1212<span class='sub'>+47 / −45</span></td><td class='num' data-sort='16.6'>16.6</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1205.0'>1205<span class='sub'>+47 / −45</span></td><td class='num' data-sort='16.5'>16.5</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1199.0'>1199<span class='sub'>+44 / −47</span></td><td class='num' data-sort='16.3'>16.3</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='12.9'>12.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+43 / −44</span></td><td class='num' data-sort='16.6'>16.6</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1193.0'>1193<span class='sub'>+51 / −52</span></td><td class='num' data-sort='13.1'>13.1</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2274.8'>2,274.8</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1175.0'>1175<span class='sub'>+55 / −56</span></td><td class='num' data-sort='9.1'>9.1</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='251.9'>251.9</td><td class='num' data-sort='6.6'>6.6</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+46 / −52</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='939.0'>939<span class='sub'>+74 / −91</span></td><td class='num' data-sort='-65.9'>-65.9</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1321.0,1271.0,1380.0,22.8,0.676,0.691,0.2,0.0
|
| 37 |
+
2,CART,Non-search,Tree,1283.0,1236.0,1331.0,19.8,0.444,0.572,0.0,0.0
|
| 38 |
+
3,MI,Non-search,Information theory,1282.0,1232.0,1331.0,19.5,0.683,0.815,0.4,0.0
|
| 39 |
+
4,Lasso,Non-search,Regularization,1268.0,1217.0,1319.0,18.7,0.389,0.73,1.8,0.0
|
| 40 |
+
5,ElasticNet,Non-search,Regularization,1252.0,1203.0,1301.0,19.0,0.481,0.777,6.3,0.0
|
| 41 |
+
6,MarkovBlanket,Search,Backward search,1248.0,1210.0,1286.0,17.8,0.413,0.447,0.1,0.0
|
| 42 |
+
7,SBE,Search,Backward search,1224.0,1171.0,1282.0,11.4,0.144,0.556,2887.2,47.4
|
| 43 |
+
8,F-Test,Non-search,Correlation,1212.0,1167.0,1259.0,16.6,0.706,0.879,0.0,0.0
|
| 44 |
+
9,GainRatio,Non-search,Information theory,1205.0,1160.0,1252.0,16.5,0.67,0.856,0.2,0.0
|
| 45 |
+
10,(R)ReliefF,Non-search,Distance,1199.0,1152.0,1243.0,16.3,0.673,0.757,12.9,0.0
|
| 46 |
+
11,mRMR,Non-search,Information theory,1197.0,1153.0,1240.0,16.6,0.614,0.66,0.4,0.0
|
| 47 |
+
12,SFS,Search,Forward search,1193.0,1141.0,1244.0,13.1,0.239,0.627,2274.8,48.8
|
| 48 |
+
13,LOCO,Search,Leave-one-out,1175.0,1119.0,1230.0,9.1,0.151,0.575,251.9,6.6
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,948.0,1046.0,0.0,-0.001,0.513,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,939.0,848.0,1013.0,-65.9,0.66,0.827,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_lightgbm/budgets_2/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1321.0,1271.0,1380.0,6.15,63.2,12.2,,0.676,0.691,0.2,0.0,71,213
|
| 3 |
-
2,CART,Non-search,Tree,1283.0,1236.0,1331.0,6.72,59.1,16.5,,0.444,0.572,0.0,0.0,71,213
|
| 4 |
-
3,MI,Non-search,Information theory,1282.0,1232.0,1331.0,6.86,58.2,15.7,,0.683,0.815,0.4,0.0,71,213
|
| 5 |
-
4,Lasso,Non-search,Regularization,1268.0,1217.0,1319.0,6.95,57.5,15.4,,0.389,0.73,1.8,0.0,71,213
|
| 6 |
-
5,ElasticNet,Non-search,Regularization,1252.0,1203.0,1301.0,7.01,57.1,16.3,,0.481,0.777,6.3,0.0,71,213
|
| 7 |
-
6,MarkovBlanket,Search,Backward search,1248.0,1210.0,1286.0,7.3,55.0,17.8,,0.413,0.447,0.1,0.0,71,213
|
| 8 |
-
7,SBE,Search,Backward search,1224.0,1171.0,1282.0,7.76,51.7,21.9,,0.144,0.556,2887.2,47.4,71,213
|
| 9 |
-
8,F-Test,Non-search,Correlation,1212.0,1167.0,1259.0,7.66,52.4,18.3,,0.706,0.879,0.0,0.0,71,213
|
| 10 |
-
9,GainRatio,Non-search,Information theory,1205.0,1160.0,1252.0,8.25,48.2,19.7,,0.67,0.856,0.2,0.0,71,213
|
| 11 |
-
10,(R)ReliefF,Non-search,Distance,1199.0,1152.0,1243.0,7.77,51.7,17.9,,0.673,0.757,12.9,0.0,71,213
|
| 12 |
-
11,mRMR,Non-search,Information theory,1197.0,1153.0,1240.0,8.29,47.9,19.2,,0.614,0.66,0.4,0.0,71,213
|
| 13 |
-
12,SFS,Search,Forward search,1193.0,1141.0,1244.0,8.11,49.2,20.8,,0.239,0.627,2274.8,48.8,71,213
|
| 14 |
-
13,LOCO,Search,Leave-one-out,1175.0,1119.0,1230.0,8.2,48.6,22.8,,0.151,0.575,251.9,6.6,71,213
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,948.0,1046.0,10.87,29.5,32.0,,-0.001,0.513,0.0,0.0,71,213
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,939.0,848.0,1013.0,12.1,20.7,37.6,,0.66,0.827,0.2,0.0,71,213
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1321.0,1271.0,1380.0,6.15,63.2,12.2,22.8,,0.676,0.691,0.2,0.0,71,213
|
| 3 |
+
2,CART,Non-search,Tree,1283.0,1236.0,1331.0,6.72,59.1,16.5,19.8,,0.444,0.572,0.0,0.0,71,213
|
| 4 |
+
3,MI,Non-search,Information theory,1282.0,1232.0,1331.0,6.86,58.2,15.7,19.5,,0.683,0.815,0.4,0.0,71,213
|
| 5 |
+
4,Lasso,Non-search,Regularization,1268.0,1217.0,1319.0,6.95,57.5,15.4,18.7,,0.389,0.73,1.8,0.0,71,213
|
| 6 |
+
5,ElasticNet,Non-search,Regularization,1252.0,1203.0,1301.0,7.01,57.1,16.3,19.0,,0.481,0.777,6.3,0.0,71,213
|
| 7 |
+
6,MarkovBlanket,Search,Backward search,1248.0,1210.0,1286.0,7.3,55.0,17.8,17.8,,0.413,0.447,0.1,0.0,71,213
|
| 8 |
+
7,SBE,Search,Backward search,1224.0,1171.0,1282.0,7.76,51.7,21.9,11.4,,0.144,0.556,2887.2,47.4,71,213
|
| 9 |
+
8,F-Test,Non-search,Correlation,1212.0,1167.0,1259.0,7.66,52.4,18.3,16.6,,0.706,0.879,0.0,0.0,71,213
|
| 10 |
+
9,GainRatio,Non-search,Information theory,1205.0,1160.0,1252.0,8.25,48.2,19.7,16.5,,0.67,0.856,0.2,0.0,71,213
|
| 11 |
+
10,(R)ReliefF,Non-search,Distance,1199.0,1152.0,1243.0,7.77,51.7,17.9,16.3,,0.673,0.757,12.9,0.0,71,213
|
| 12 |
+
11,mRMR,Non-search,Information theory,1197.0,1153.0,1240.0,8.29,47.9,19.2,16.6,,0.614,0.66,0.4,0.0,71,213
|
| 13 |
+
12,SFS,Search,Forward search,1193.0,1141.0,1244.0,8.11,49.2,20.8,13.1,,0.239,0.627,2274.8,48.8,71,213
|
| 14 |
+
13,LOCO,Search,Leave-one-out,1175.0,1119.0,1230.0,8.2,48.6,22.8,9.1,,0.151,0.575,251.9,6.6,71,213
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,948.0,1046.0,10.87,29.5,32.0,0.0,,-0.001,0.513,0.0,0.0,71,213
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,939.0,848.0,1013.0,12.1,20.7,37.6,-65.9,,0.66,0.827,0.2,0.0,71,213
|
data/subsets/tasks_all/models_lightgbm/budgets_3/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+58 / −49</span></td><td class='num' data-sort='5.82'>5.82</td><td class='num' data-sort='11.0'>11.0</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1210.0'>1210<span class='sub'>+48 / −41</span></td><td class='num' data-sort='6.66'>6.66</td><td class='num' data-sort='14.0'>14.0</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1207.0'>1207<span class='sub'>+46 / −47</span></td><td class='num' data-sort='6.9'>6.90</td><td class='num' data-sort='13.4'>13.4</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1207.0'>1207<span class='sub'>+54 / −53</span></td><td class='num' data-sort='6.79'>6.79</td><td class='num' data-sort='14.1'>14.1</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+48 / −49</span></td><td class='num' data-sort='7.19'>7.19</td><td class='num' data-sort='14.4'>14.4</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1183.0'>1183<span class='sub'>+44 / −48</span></td><td class='num' data-sort='7.42'>7.42</td><td class='num' data-sort='13.5'>13.5</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1153.0'>1153<span class='sub'>+43 / −43</span></td><td class='num' data-sort='8.12'>8.12</td><td class='num' data-sort='15.4'>15.4</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1145.0'>1145<span class='sub'>+52 / −50</span></td><td class='num' data-sort='7.9'>7.90</td><td class='num' data-sort='14.4'>14.4</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='13.1'>13.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1128.0'>1128<span class='sub'>+43 / −43</span></td><td class='num' data-sort='8.14'>8.14</td><td class='num' data-sort='16.5'>16.5</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1122.0'>1122<span class='sub'>+57 / −58</span></td><td class='num' data-sort='7.98'>7.98</td><td class='num' data-sort='18.4'>18.4</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1923.4'>1,923.4</td><td class='num' data-sort='44.6'>44.6</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1118.0'>1118<span class='sub'>+53 / −53</span></td><td class='num' data-sort='8.21'>8.21</td><td class='num' data-sort='19.1'>19.1</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='236.6'>236.6</td><td class='num' data-sort='11.3'>11.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1115.0'>1115<span class='sub'>+46 / −44</span></td><td class='num' data-sort='8.49'>8.49</td><td class='num' data-sort='16.2'>16.2</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1098.0'>1098<span class='sub'>+50 / −48</span></td><td class='num' data-sort='8.54'>8.54</td><td class='num' data-sort='19.2'>19.2</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2771.3'>2,771.3</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+50 / −57</span></td><td class='num' data-sort='9.97'>9.97</td><td class='num' data-sort='24.9'>24.9</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='878.0'>878<span class='sub'>+72 / −82</span></td><td class='num' data-sort='11.86'>11.86</td><td class='num' data-sort='31.8'>31.8</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1273.0,1224.0,1331.0,
|
| 37 |
-
2,CART,Non-search,Tree,1210.0,1169.0,1258.0,
|
| 38 |
-
3,Lasso,Non-search,Regularization,1207.0,1160.0,1253.0,
|
| 39 |
-
4,ElasticNet,Non-search,Regularization,1207.0,1154.0,1261.0,
|
| 40 |
-
5,MI,Non-search,Information theory,1197.0,1148.0,1245.0,
|
| 41 |
-
6,F-Test,Non-search,Correlation,1183.0,1135.0,1227.0,
|
| 42 |
-
7,mRMR,Non-search,Information theory,1153.0,1110.0,1196.0,
|
| 43 |
-
8,(R)ReliefF,Non-search,Distance,1145.0,1095.0,1197.0,
|
| 44 |
-
9,MarkovBlanket,Search,Backward search,1128.0,1085.0,1171.0,
|
| 45 |
-
10,SBE,Search,Backward search,1122.0,1064.0,1179.0,
|
| 46 |
-
11,LOCO,Search,Leave-one-out,1118.0,1065.0,1171.0,
|
| 47 |
-
12,GainRatio,Non-search,Information theory,1115.0,1071.0,1161.0,
|
| 48 |
-
13,SFS,Search,Forward search,1098.0,1050.0,1148.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,943.0,1050.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,878.0,796.0,950.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1273.0'>1273<span class='sub'>+58 / −49</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1210.0'>1210<span class='sub'>+48 / −41</span></td><td class='num' data-sort='14.3'>14.3</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1207.0'>1207<span class='sub'>+46 / −47</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1207.0'>1207<span class='sub'>+54 / −53</span></td><td class='num' data-sort='13.3'>13.3</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+48 / −49</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1183.0'>1183<span class='sub'>+44 / −48</span></td><td class='num' data-sort='14.2'>14.2</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1153.0'>1153<span class='sub'>+43 / −43</span></td><td class='num' data-sort='13.5'>13.5</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1145.0'>1145<span class='sub'>+52 / −50</span></td><td class='num' data-sort='12.8'>12.8</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='13.1'>13.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1128.0'>1128<span class='sub'>+43 / −43</span></td><td class='num' data-sort='12.3'>12.3</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1122.0'>1122<span class='sub'>+57 / −58</span></td><td class='num' data-sort='5.8'>5.8</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1923.4'>1,923.4</td><td class='num' data-sort='44.6'>44.6</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1118.0'>1118<span class='sub'>+53 / −53</span></td><td class='num' data-sort='6.2'>6.2</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='236.6'>236.6</td><td class='num' data-sort='11.3'>11.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1115.0'>1115<span class='sub'>+46 / −44</span></td><td class='num' data-sort='12.3'>12.3</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1098.0'>1098<span class='sub'>+50 / −48</span></td><td class='num' data-sort='7.3'>7.3</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2771.3'>2,771.3</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+50 / −57</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='878.0'>878<span class='sub'>+72 / −82</span></td><td class='num' data-sort='-126.6'>-126.6</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1273.0,1224.0,1331.0,17.7,0.661,0.6,0.2,0.0
|
| 37 |
+
2,CART,Non-search,Tree,1210.0,1169.0,1258.0,14.3,0.402,0.507,0.0,0.0
|
| 38 |
+
3,Lasso,Non-search,Regularization,1207.0,1160.0,1253.0,13.7,0.334,0.643,1.9,0.0
|
| 39 |
+
4,ElasticNet,Non-search,Regularization,1207.0,1154.0,1261.0,13.3,0.419,0.665,6.1,0.0
|
| 40 |
+
5,MI,Non-search,Information theory,1197.0,1148.0,1245.0,13.7,0.649,0.723,0.4,0.0
|
| 41 |
+
6,F-Test,Non-search,Correlation,1183.0,1135.0,1227.0,14.2,0.668,0.781,0.0,0.0
|
| 42 |
+
7,mRMR,Non-search,Information theory,1153.0,1110.0,1196.0,13.5,0.656,0.593,0.5,7.0
|
| 43 |
+
8,(R)ReliefF,Non-search,Distance,1145.0,1095.0,1197.0,12.8,0.656,0.667,13.1,0.0
|
| 44 |
+
9,MarkovBlanket,Search,Backward search,1128.0,1085.0,1171.0,12.3,0.434,0.473,0.1,0.0
|
| 45 |
+
10,SBE,Search,Backward search,1122.0,1064.0,1179.0,5.8,0.095,0.53,1923.4,44.6
|
| 46 |
+
11,LOCO,Search,Leave-one-out,1118.0,1065.0,1171.0,6.2,0.114,0.544,236.6,11.3
|
| 47 |
+
12,GainRatio,Non-search,Information theory,1115.0,1071.0,1161.0,12.3,0.641,0.751,0.2,0.0
|
| 48 |
+
13,SFS,Search,Forward search,1098.0,1050.0,1148.0,7.3,0.225,0.568,2771.3,48.4
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,943.0,1050.0,0.0,0.002,0.494,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,878.0,796.0,950.0,-126.6,0.689,0.758,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_lightgbm/budgets_3/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='173.5' x2='173.5' y1='16.2' y2='368.0'/><text class='sa-text' x='173.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='271.4' x2='271.4' y1='16.2' y2='368.0'/><text class='sa-text' x='271.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='369.4' x2='369.4' y1='16.2' y2='368.0'/><text class='sa-text' x='369.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='467.4' x2='467.4' y1='16.2' y2='368.0'/><text class='sa-text' x='467.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='565.4' x2='565.4' y1='16.2' y2='368.0'/><text class='sa-text' x='565.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.4' x2='663.4' y1='16.2' y2='368.0'/><text class='sa-text' x='663.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.4' x2='761.4' y1='16.2' y2='368.0'/><text class='sa-text' x='761.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='81.3' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='72.3' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='81.3' x2='847.8' y1='306.8' y2='306.8'/><text class='sa-text' x='72.3' y='312.8' text-anchor='end' font-size='15'>2.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='261.6' y2='261.6'/><text class='sa-text' x='72.3' y='267.6' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='216.4' y2='216.4'/><text class='sa-text' x='72.3' y='222.4' text-anchor='end' font-size='15'>7.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='171.1' y2='171.1'/><text class='sa-text' x='72.3' y='177.1' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='81.3' x2='847.8' y1='125.9' y2='125.9'/><text class='sa-text' x='72.3' y='131.9' text-anchor='end' font-size='15'>12.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='80.7' y2='80.7'/><text class='sa-text' x='72.3' y='86.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='81.3' x2='847.8' y1='35.4' y2='35.4'/><text class='sa-text' x='72.3' y='41.4' text-anchor='end' font-size='15'>17.5</text><rect class='sa-frame-line' x='81.3' y='16.2' width='766.5' height='351.8'/><text class='sa-text' x='464.5' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='123.6' y1='51.9' x2='83.3' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='96.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='116.1,352.0 350.8,96.0 483.0,93.4 524.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='524.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='483.0' cy='93.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='350.8' cy='96.0' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='532.7' cy='103.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='692.7' cy='104.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='547.5' cy='108.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='111.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='798.3' cy='120.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='477.6' cy='129.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='494.6' cy='130.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='813.0' cy='220.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='742.2' cy='240.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='799.0' cy='247.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='116.1' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='524.8' y1='32.2' x2='646.6' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='646.6' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><text class='sa-text sa-halo' x='496.6' y='81.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='350.8' y1='96.0' x2='301.4' y2='109.2' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='301.4' y='109.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='532.7' y1='103.7' x2='566.7' y2='63.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='566.7' y='63.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='692.7' y1='104.6' x2='715.7' y2='89.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='715.7' y='89.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='547.5' y1='108.5' x2='598.1' y2='96.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='598.1' y='96.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='695.4' y1='111.1' x2='721.0' y2='127.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='721.0' y='127.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='798.3' y1='120.0' x2='797.9' y2='163.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='797.9' y='163.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='477.6' y1='129.1' x2='397.2' y2='136.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='397.2' y='136.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='494.6' y1='130.1' x2='582.9' y2='175.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='582.9' y='175.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='813.0' y1='220.6' x2='828.9' y2='206.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.9' y='206.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='742.2' y1='240.7' x2='710.0' y2='254.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='710.0' y='254.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='799.0' y1='247.0' x2='825.6' y2='261.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='825.6' y='261.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='116.1' y1='352.0' x2='168.9' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='168.9' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='524.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 17.7% Selection time per 1k features: 38.5 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 17.7% Selection time per 1k features: 38.5 s Datasets: 71'/><circle class='sa-hit' cx='483.0' cy='93.4' r='10.5' data-tip='CART (on the Pareto front) Improvement over Random: 14.3% Selection time per 1k features: 14.4 s Datasets: 71' aria-label='CART (on the Pareto front) Improvement over Random: 14.3% Selection time per 1k features: 14.4 s Datasets: 71'/><circle class='sa-hit' cx='350.8' cy='96.0' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 14.2% Selection time per 1k features: 0.645 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 14.2% Selection time per 1k features: 0.645 s Datasets: 71'/><circle class='sa-hit' cx='532.7' cy='103.7' r='10.5' data-tip='MI Improvement over Random: 13.7% Selection time per 1k features: 46.4 s Datasets: 71 Next to mRMR: 13.5%, 65.6 s' aria-label='MI Improvement over Random: 13.7% Selection time per 1k features: 46.4 s Datasets: 71 Next to mRMR: 13.5%, 65.6 s'/><circle class='sa-hit' cx='692.7' cy='104.6' r='10.5' data-tip='Lasso Improvement over Random: 13.7% Selection time per 1k features: 1.99e+03 s Datasets: 71 Next to ElasticNet: 13.3%, 2.12e+03 s' aria-label='Lasso Improvement over Random: 13.7% Selection time per 1k features: 1.99e+03 s Datasets: 71 Next to ElasticNet: 13.3%, 2.12e+03 s'/><circle class='sa-hit' cx='547.5' cy='108.5' r='10.5' data-tip='mRMR Improvement over Random: 13.5% Selection time per 1k features: 65.6 s Datasets: 71 Next to MI: 13.7%, 46.4 s' aria-label='mRMR Improvement over Random: 13.5% Selection time per 1k features: 65.6 s Datasets: 71 Next to MI: 13.7%, 46.4 s'/><circle class='sa-hit' cx='695.4' cy='111.1' r='10.5' data-tip='ElasticNet Improvement over Random: 13.3% Selection time per 1k features: 2.12e+03 s Datasets: 71 Next to Lasso: 13.7%, 1.99e+03 s' aria-label='ElasticNet Improvement over Random: 13.3% Selection time per 1k features: 2.12e+03 s Datasets: 71 Next to Lasso: 13.7%, 1.99e+03 s'/><circle class='sa-hit' cx='798.3' cy='120.0' r='10.5' data-tip='(R)ReliefF Improvement over Random: 12.8% Selection time per 1k features: 2.38e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 12.8% Selection time per 1k features: 2.38e+04 s Datasets: 71'/><circle class='sa-hit' cx='477.6' cy='129.1' r='10.5' data-tip='GainRatio Improvement over Random: 12.3% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 12.3%, 18.9 s' aria-label='GainRatio Improvement over Random: 12.3% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 12.3%, 18.9 s'/><circle class='sa-hit' cx='494.6' cy='130.1' r='10.5' data-tip='MarkovBlanket Improvement over Random: 12.3% Selection time per 1k features: 18.9 s Datasets: 71 Next to GainRatio: 12.3%, 12.7 s' aria-label='MarkovBlanket Improvement over Random: 12.3% Selection time per 1k features: 18.9 s Datasets: 71 Next to GainRatio: 12.3%, 12.7 s'/><circle class='sa-hit' cx='813.0' cy='220.6' r='10.5' data-tip='SFS Improvement over Random: 7.3% Selection time per 1k features: 3.36e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 7.3% Selection time per 1k features: 3.36e+04 s Datasets: 71'/><circle class='sa-hit' cx='742.2' cy='240.7' r='10.5' data-tip='LOCO Improvement over Random: 6.2% Selection time per 1k features: 6.37e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 6.2% Selection time per 1k features: 6.37e+03 s Datasets: 71'/><circle class='sa-hit' cx='799.0' cy='247.0' r='10.5' data-tip='SBE Improvement over Random: 5.8% Selection time per 1k features: 2.42e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 5.8% Selection time per 1k features: 2.42e+04 s Datasets: 71'/><circle class='sa-hit' cx='116.1' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,17.68,38.5146,71,True
|
| 37 |
CART,14.296,14.4215,71,True
|
| 38 |
F-Test,14.1513,0.6455,71,True
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='173.5' x2='173.5' y1='16.2' y2='368.0'/><text class='sa-text' x='173.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='271.4' x2='271.4' y1='16.2' y2='368.0'/><text class='sa-text' x='271.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='369.4' x2='369.4' y1='16.2' y2='368.0'/><text class='sa-text' x='369.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='467.4' x2='467.4' y1='16.2' y2='368.0'/><text class='sa-text' x='467.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='565.4' x2='565.4' y1='16.2' y2='368.0'/><text class='sa-text' x='565.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.4' x2='663.4' y1='16.2' y2='368.0'/><text class='sa-text' x='663.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='761.4' x2='761.4' y1='16.2' y2='368.0'/><text class='sa-text' x='761.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='81.3' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='72.3' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='81.3' x2='847.8' y1='306.8' y2='306.8'/><text class='sa-text' x='72.3' y='312.8' text-anchor='end' font-size='15'>2.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='261.6' y2='261.6'/><text class='sa-text' x='72.3' y='267.6' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='216.4' y2='216.4'/><text class='sa-text' x='72.3' y='222.4' text-anchor='end' font-size='15'>7.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='171.1' y2='171.1'/><text class='sa-text' x='72.3' y='177.1' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='81.3' x2='847.8' y1='125.9' y2='125.9'/><text class='sa-text' x='72.3' y='131.9' text-anchor='end' font-size='15'>12.5</text><line class='sa-grid' x1='81.3' x2='847.8' y1='80.7' y2='80.7'/><text class='sa-text' x='72.3' y='86.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='81.3' x2='847.8' y1='35.4' y2='35.4'/><text class='sa-text' x='72.3' y='41.4' text-anchor='end' font-size='15'>17.5</text><rect class='sa-frame-line' x='81.3' y='16.2' width='766.5' height='351.8'/><text class='sa-text' x='464.5' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='123.6' y1='51.9' x2='83.3' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='96.6' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='116.1,352.0 350.8,96.0 483.0,93.4 524.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='524.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='483.0' cy='93.4' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='350.8' cy='96.0' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='532.7' cy='103.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='692.7' cy='104.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='547.5' cy='108.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='111.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='798.3' cy='120.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='477.6' cy='129.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='494.6' cy='130.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='813.0' cy='220.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='742.2' cy='240.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='799.0' cy='247.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='116.1' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='524.8' y1='32.2' x2='647.2' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='647.2' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='483.0' y1='93.4' x2='494.7' y2='75.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='494.7' y='75.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><line class='sa-ink' x1='350.8' y1='96.0' x2='301.4' y2='106.9' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='301.4' y='106.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='532.7' y1='103.7' x2='563.6' y2='66.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='563.6' y='66.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='692.7' y1='104.6' x2='715.7' y2='89.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='715.7' y='89.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='547.5' y1='108.5' x2='601.8' y2='97.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='601.8' y='97.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='695.4' y1='111.1' x2='720.3' y2='133.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='720.3' y='133.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='798.3' y1='120.0' x2='797.9' y2='161.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='797.9' y='161.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='477.6' y1='129.1' x2='402.2' y2='137.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='402.2' y='137.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='494.6' y1='130.1' x2='581.7' y2='174.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='581.7' y='174.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='813.0' y1='220.6' x2='828.9' y2='206.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.9' y='206.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='742.2' y1='240.7' x2='708.8' y2='255.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='708.8' y='255.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='799.0' y1='247.0' x2='825.0' y2='260.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='825.0' y='260.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='116.1' y1='352.0' x2='168.9' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='168.9' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='524.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 17.7% Selection time per 1k features: 38.5 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 17.7% Selection time per 1k features: 38.5 s Datasets: 71'/><circle class='sa-hit' cx='483.0' cy='93.4' r='10.5' data-tip='CART (on the Pareto front) Improvement over Random: 14.3% Selection time per 1k features: 14.4 s Datasets: 71' aria-label='CART (on the Pareto front) Improvement over Random: 14.3% Selection time per 1k features: 14.4 s Datasets: 71'/><circle class='sa-hit' cx='350.8' cy='96.0' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 14.2% Selection time per 1k features: 0.645 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 14.2% Selection time per 1k features: 0.645 s Datasets: 71'/><circle class='sa-hit' cx='532.7' cy='103.7' r='10.5' data-tip='MI Improvement over Random: 13.7% Selection time per 1k features: 46.4 s Datasets: 71 Next to mRMR: 13.5%, 65.6 s' aria-label='MI Improvement over Random: 13.7% Selection time per 1k features: 46.4 s Datasets: 71 Next to mRMR: 13.5%, 65.6 s'/><circle class='sa-hit' cx='692.7' cy='104.6' r='10.5' data-tip='Lasso Improvement over Random: 13.7% Selection time per 1k features: 1.99e+03 s Datasets: 71 Next to ElasticNet: 13.3%, 2.12e+03 s' aria-label='Lasso Improvement over Random: 13.7% Selection time per 1k features: 1.99e+03 s Datasets: 71 Next to ElasticNet: 13.3%, 2.12e+03 s'/><circle class='sa-hit' cx='547.5' cy='108.5' r='10.5' data-tip='mRMR Improvement over Random: 13.5% Selection time per 1k features: 65.6 s Datasets: 71 Next to MI: 13.7%, 46.4 s' aria-label='mRMR Improvement over Random: 13.5% Selection time per 1k features: 65.6 s Datasets: 71 Next to MI: 13.7%, 46.4 s'/><circle class='sa-hit' cx='695.4' cy='111.1' r='10.5' data-tip='ElasticNet Improvement over Random: 13.3% Selection time per 1k features: 2.12e+03 s Datasets: 71 Next to Lasso: 13.7%, 1.99e+03 s' aria-label='ElasticNet Improvement over Random: 13.3% Selection time per 1k features: 2.12e+03 s Datasets: 71 Next to Lasso: 13.7%, 1.99e+03 s'/><circle class='sa-hit' cx='798.3' cy='120.0' r='10.5' data-tip='(R)ReliefF Improvement over Random: 12.8% Selection time per 1k features: 2.38e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 12.8% Selection time per 1k features: 2.38e+04 s Datasets: 71'/><circle class='sa-hit' cx='477.6' cy='129.1' r='10.5' data-tip='GainRatio Improvement over Random: 12.3% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 12.3%, 18.9 s' aria-label='GainRatio Improvement over Random: 12.3% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 12.3%, 18.9 s'/><circle class='sa-hit' cx='494.6' cy='130.1' r='10.5' data-tip='MarkovBlanket Improvement over Random: 12.3% Selection time per 1k features: 18.9 s Datasets: 71 Next to GainRatio: 12.3%, 12.7 s' aria-label='MarkovBlanket Improvement over Random: 12.3% Selection time per 1k features: 18.9 s Datasets: 71 Next to GainRatio: 12.3%, 12.7 s'/><circle class='sa-hit' cx='813.0' cy='220.6' r='10.5' data-tip='SFS Improvement over Random: 7.3% Selection time per 1k features: 3.36e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 7.3% Selection time per 1k features: 3.36e+04 s Datasets: 71'/><circle class='sa-hit' cx='742.2' cy='240.7' r='10.5' data-tip='LOCO Improvement over Random: 6.2% Selection time per 1k features: 6.37e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 6.2% Selection time per 1k features: 6.37e+03 s Datasets: 71'/><circle class='sa-hit' cx='799.0' cy='247.0' r='10.5' data-tip='SBE Improvement over Random: 5.8% Selection time per 1k features: 2.42e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 5.8% Selection time per 1k features: 2.42e+04 s Datasets: 71'/><circle class='sa-hit' cx='116.1' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,17.68,38.5146,71,True
|
| 37 |
CART,14.296,14.4215,71,True
|
| 38 |
F-Test,14.1513,0.6455,71,True
|
data/subsets/tasks_all/models_lightgbm/budgets_3/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1273.0,1224.0,1331.0,5.82,65.6,11.0,,0.661,0.6,0.2,0.0,71,213
|
| 3 |
-
2,CART,Non-search,Tree,1210.0,1169.0,1258.0,6.66,59.6,14.0,,0.402,0.507,0.0,0.0,71,213
|
| 4 |
-
3,Lasso,Non-search,Regularization,1207.0,1160.0,1253.0,6.9,57.9,13.4,,0.334,0.643,1.9,0.0,71,213
|
| 5 |
-
4,ElasticNet,Non-search,Regularization,1207.0,1154.0,1261.0,6.79,58.6,14.1,,0.419,0.665,6.1,0.0,71,213
|
| 6 |
-
5,MI,Non-search,Information theory,1197.0,1148.0,1245.0,7.19,55.8,14.4,,0.649,0.723,0.4,0.0,71,213
|
| 7 |
-
6,F-Test,Non-search,Correlation,1183.0,1135.0,1227.0,7.42,54.1,13.5,,0.668,0.781,0.0,0.0,71,213
|
| 8 |
-
7,mRMR,Non-search,Information theory,1153.0,1110.0,1196.0,8.12,49.1,15.4,,0.656,0.593,0.5,7.0,71,213
|
| 9 |
-
8,(R)ReliefF,Non-search,Distance,1145.0,1095.0,1197.0,7.9,50.7,14.4,,0.656,0.667,13.1,0.0,71,213
|
| 10 |
-
9,MarkovBlanket,Search,Backward search,1128.0,1085.0,1171.0,8.14,49.0,16.5,,0.434,0.473,0.1,0.0,71,213
|
| 11 |
-
10,SBE,Search,Backward search,1122.0,1064.0,1179.0,7.98,50.1,18.4,,0.095,0.53,1923.4,44.6,71,213
|
| 12 |
-
11,LOCO,Search,Leave-one-out,1118.0,1065.0,1171.0,8.21,48.5,19.1,,0.114,0.544,236.6,11.3,71,213
|
| 13 |
-
12,GainRatio,Non-search,Information theory,1115.0,1071.0,1161.0,8.49,46.5,16.2,,0.641,0.751,0.2,0.0,71,213
|
| 14 |
-
13,SFS,Search,Forward search,1098.0,1050.0,1148.0,8.54,46.1,19.2,,0.225,0.568,2771.3,48.4,71,213
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,943.0,1050.0,9.97,35.9,24.9,,0.002,0.494,0.0,0.0,71,213
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,878.0,796.0,950.0,11.86,22.4,31.8,,0.689,0.758,0.2,0.0,71,213
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1273.0,1224.0,1331.0,5.82,65.6,11.0,17.7,,0.661,0.6,0.2,0.0,71,213
|
| 3 |
+
2,CART,Non-search,Tree,1210.0,1169.0,1258.0,6.66,59.6,14.0,14.3,,0.402,0.507,0.0,0.0,71,213
|
| 4 |
+
3,Lasso,Non-search,Regularization,1207.0,1160.0,1253.0,6.9,57.9,13.4,13.7,,0.334,0.643,1.9,0.0,71,213
|
| 5 |
+
4,ElasticNet,Non-search,Regularization,1207.0,1154.0,1261.0,6.79,58.6,14.1,13.3,,0.419,0.665,6.1,0.0,71,213
|
| 6 |
+
5,MI,Non-search,Information theory,1197.0,1148.0,1245.0,7.19,55.8,14.4,13.7,,0.649,0.723,0.4,0.0,71,213
|
| 7 |
+
6,F-Test,Non-search,Correlation,1183.0,1135.0,1227.0,7.42,54.1,13.5,14.2,,0.668,0.781,0.0,0.0,71,213
|
| 8 |
+
7,mRMR,Non-search,Information theory,1153.0,1110.0,1196.0,8.12,49.1,15.4,13.5,,0.656,0.593,0.5,7.0,71,213
|
| 9 |
+
8,(R)ReliefF,Non-search,Distance,1145.0,1095.0,1197.0,7.9,50.7,14.4,12.8,,0.656,0.667,13.1,0.0,71,213
|
| 10 |
+
9,MarkovBlanket,Search,Backward search,1128.0,1085.0,1171.0,8.14,49.0,16.5,12.3,,0.434,0.473,0.1,0.0,71,213
|
| 11 |
+
10,SBE,Search,Backward search,1122.0,1064.0,1179.0,7.98,50.1,18.4,5.8,,0.095,0.53,1923.4,44.6,71,213
|
| 12 |
+
11,LOCO,Search,Leave-one-out,1118.0,1065.0,1171.0,8.21,48.5,19.1,6.2,,0.114,0.544,236.6,11.3,71,213
|
| 13 |
+
12,GainRatio,Non-search,Information theory,1115.0,1071.0,1161.0,8.49,46.5,16.2,12.3,,0.641,0.751,0.2,0.0,71,213
|
| 14 |
+
13,SFS,Search,Forward search,1098.0,1050.0,1148.0,8.54,46.1,19.2,7.3,,0.225,0.568,2771.3,48.4,71,213
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,943.0,1050.0,9.97,35.9,24.9,0.0,,0.002,0.494,0.0,0.0,71,213
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,878.0,796.0,950.0,11.86,22.4,31.8,-126.6,,0.689,0.758,0.2,0.0,71,213
|
data/subsets/tasks_all/models_lightgbm/budgets_4/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+52 / −44</span></td><td class='num' data-sort='5.9'>5.90</td><td class='num' data-sort='8.7'>8.7</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1177.0'>1177<span class='sub'>+57 / −49</span></td><td class='num' data-sort='6.9'>6.90</td><td class='num' data-sort='11.0'>11.0</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+49 / −49</span></td><td class='num' data-sort='7.05'>7.05</td><td class='num' data-sort='10.8'>10.8</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+46 / −41</span></td><td class='num' data-sort='7.22'>7.22</td><td class='num' data-sort='10.5'>10.5</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1160.0'>1160<span class='sub'>+44 / −41</span></td><td class='num' data-sort='7.29'>7.29</td><td class='num' data-sort='11.4'>11.4</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1154.0'>1154<span class='sub'>+45 / −50</span></td><td class='num' data-sort='7.5'>7.50</td><td class='num' data-sort='10.2'>10.2</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1148.0'>1148<span class='sub'>+45 / −48</span></td><td class='num' data-sort='7.86'>7.86</td><td class='num' data-sort='11.2'>11.2</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1125.0'>1125<span class='sub'>+45 / −48</span></td><td class='num' data-sort='7.75'>7.75</td><td class='num' data-sort='10.2'>10.2</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1116.0'>1116<span class='sub'>+46 / −48</span></td><td class='num' data-sort='8.39'>8.39</td><td class='num' data-sort='11.3'>11.3</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1112.0'>1112<span class='sub'>+51 / −53</span></td><td class='num' data-sort='8.4'>8.40</td><td class='num' data-sort='13.1'>13.1</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='1001.8'>1,001.8</td><td class='num' data-sort='40.8'>40.8</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+50 / −49</span></td><td class='num' data-sort='8.7'>8.70</td><td class='num' data-sort='13.1'>13.1</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='3227.2'>3,227.2</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1092.0'>1092<span class='sub'>+53 / −50</span></td><td class='num' data-sort='8.36'>8.36</td><td class='num' data-sort='13.8'>13.8</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='249.4'>249.4</td><td class='num' data-sort='18.3'>18.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1091.0'>1091<span class='sub'>+41 / −46</span></td><td class='num' data-sort='8.61'>8.61</td><td class='num' data-sort='12.7'>12.7</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+53 / −56</span></td><td class='num' data-sort='9.8'>9.80</td><td class='num' data-sort='16.9'>16.9</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='968.0'>968<span class='sub'>+56 / −74</span></td><td class='num' data-sort='10.28'>10.28</td><td class='num' data-sort='20.2'>20.2</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1274.0,1230.0,1326.0,
|
| 37 |
-
2,CART,Non-search,Tree,1177.0,1128.0,1234.0,
|
| 38 |
-
3,ElasticNet,Non-search,Regularization,1171.0,1122.0,1220.0,
|
| 39 |
-
4,Lasso,Non-search,Regularization,1169.0,1128.0,1215.0,7.
|
| 40 |
-
5,MI,Non-search,Information theory,1160.0,1119.0,1204.0,
|
| 41 |
-
6,F-Test,Non-search,Correlation,1154.0,1104.0,1199.0,
|
| 42 |
-
7,(R)ReliefF,Non-search,Distance,1148.0,1100.0,1193.0,7.
|
| 43 |
-
8,mRMR,Non-search,Information theory,1125.0,1077.0,1170.0,
|
| 44 |
-
9,MarkovBlanket,Search,Backward search,1116.0,1068.0,1162.0,
|
| 45 |
-
10,SBE,Search,Backward search,1112.0,1059.0,1163.0,
|
| 46 |
-
11,SFS,Search,Forward search,1097.0,1048.0,1147.0,
|
| 47 |
-
12,LOCO,Search,Leave-one-out,1092.0,1042.0,1145.0,
|
| 48 |
-
13,GainRatio,Non-search,Information theory,1091.0,1045.0,1132.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,944.0,1053.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,968.0,894.0,1024.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+52 / −44</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1177.0'>1177<span class='sub'>+57 / −49</span></td><td class='num' data-sort='7.7'>7.7</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1171.0'>1171<span class='sub'>+49 / −49</span></td><td class='num' data-sort='6.3'>6.3</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+46 / −41</span></td><td class='num' data-sort='7.6'>7.6</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1160.0'>1160<span class='sub'>+44 / −41</span></td><td class='num' data-sort='4.6'>4.6</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1154.0'>1154<span class='sub'>+45 / −50</span></td><td class='num' data-sort='9.2'>9.2</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1148.0'>1148<span class='sub'>+45 / −48</span></td><td class='num' data-sort='7.0'>7.0</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1125.0'>1125<span class='sub'>+45 / −48</span></td><td class='num' data-sort='8.3'>8.3</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1116.0'>1116<span class='sub'>+46 / −48</span></td><td class='num' data-sort='7.1'>7.1</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1112.0'>1112<span class='sub'>+51 / −53</span></td><td class='num' data-sort='5.3'>5.3</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='1001.8'>1,001.8</td><td class='num' data-sort='40.8'>40.8</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1097.0'>1097<span class='sub'>+50 / −49</span></td><td class='num' data-sort='4.0'>4.0</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='3227.2'>3,227.2</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1092.0'>1092<span class='sub'>+53 / −50</span></td><td class='num' data-sort='1.2'>1.2</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='249.4'>249.4</td><td class='num' data-sort='18.3'>18.3</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1091.0'>1091<span class='sub'>+41 / −46</span></td><td class='num' data-sort='5.1'>5.1</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+53 / −56</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='968.0'>968<span class='sub'>+56 / −74</span></td><td class='num' data-sort='-220.2'>-220.2</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1274.0,1230.0,1326.0,11.1,0.597,0.507,0.2,0.0
|
| 37 |
+
2,CART,Non-search,Tree,1177.0,1128.0,1234.0,7.7,0.324,0.482,0.1,0.0
|
| 38 |
+
3,ElasticNet,Non-search,Regularization,1171.0,1122.0,1220.0,6.3,0.308,0.568,6.1,0.0
|
| 39 |
+
4,Lasso,Non-search,Regularization,1169.0,1128.0,1215.0,7.6,0.246,0.563,1.8,0.0
|
| 40 |
+
5,MI,Non-search,Information theory,1160.0,1119.0,1204.0,4.6,0.587,0.627,0.4,0.0
|
| 41 |
+
6,F-Test,Non-search,Correlation,1154.0,1104.0,1199.0,9.2,0.562,0.689,0.0,0.0
|
| 42 |
+
7,(R)ReliefF,Non-search,Distance,1148.0,1100.0,1193.0,7.0,0.612,0.594,13.2,0.0
|
| 43 |
+
8,mRMR,Non-search,Information theory,1125.0,1077.0,1170.0,8.3,0.651,0.53,0.5,9.9
|
| 44 |
+
9,MarkovBlanket,Search,Backward search,1116.0,1068.0,1162.0,7.1,0.404,0.463,0.1,0.0
|
| 45 |
+
10,SBE,Search,Backward search,1112.0,1059.0,1163.0,5.3,0.059,0.517,1001.8,40.8
|
| 46 |
+
11,SFS,Search,Forward search,1097.0,1048.0,1147.0,4.0,0.176,0.525,3227.2,48.4
|
| 47 |
+
12,LOCO,Search,Leave-one-out,1092.0,1042.0,1145.0,1.2,0.08,0.52,249.4,18.3
|
| 48 |
+
13,GainRatio,Non-search,Information theory,1091.0,1045.0,1132.0,5.1,0.55,0.659,0.2,0.0
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,944.0,1053.0,0.0,0.002,0.5,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,968.0,894.0,1024.0,-220.2,0.678,0.701,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_lightgbm/budgets_4/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='159.1' x2='159.1' y1='16.2' y2='368.0'/><text class='sa-text' x='159.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='258.8' x2='258.8' y1='16.2' y2='368.0'/><text class='sa-text' x='258.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='358.5' x2='358.5' y1='16.2' y2='368.0'/><text class='sa-text' x='358.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='458.2' x2='458.2' y1='16.2' y2='368.0'/><text class='sa-text' x='458.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='558.0' x2='558.0' y1='16.2' y2='368.0'/><text class='sa-text' x='558.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.7' x2='657.7' y1='16.2' y2='368.0'/><text class='sa-text' x='657.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.4' x2='757.4' y1='16.2' y2='368.0'/><text class='sa-text' x='757.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='294.4' y2='294.4'/><text class='sa-text' x='57.9' y='300.4' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='236.8' y2='236.8'/><text class='sa-text' x='57.9' y='242.8' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='179.1' y2='179.1'/><text class='sa-text' x='57.9' y='185.1' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='121.5' y2='121.5'/><text class='sa-text' x='57.9' y='127.5' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='63.8' y2='63.8'/><text class='sa-text' x='57.9' y='69.8' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 338.2,88.2 516.9,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.9' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='338.2' cy='88.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='563.3' cy='114.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.4' cy='129.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='688.9' cy='134.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='483.5' cy='146.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.1' cy='149.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='689.0' cy='169.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='776.9' cy='199.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.0' cy='206.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='524.8' cy='219.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='237.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.3' cy='318.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.9' y1='32.2' x2='410.5' y2='46.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='410.5' y='46.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='338.2' y1='88.2' x2='298.6' y2='102.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='298.6' y='102.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='563.3' y1='114.1' x2='611.1' y2='127.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='611.1' y='127.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='474.4' y1='129.0' x2='495.8' y2='114.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='495.8' y='114.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='688.9' y1='134.3' x2='712.8' y2='106.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='712.8' y='106.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='483.5' y1='146.4' x2='593.7' y2='152.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='593.7' y='152.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><text class='sa-dim sa-halo' x='796.9' y='135.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><text class='sa-dim sa-halo' x='697.9' y='185.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='776.9' y1='199.6' x2='815.3' y2='195.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='815.3' y='195.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='468.0' y1='206.5' x2='451.0' y2='191.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='451.0' y='191.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='545.0' y='224.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='812.3' y1='237.0' x2='828.6' y2='251.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='251.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='737.3' y1='318.6' x2='703.3' y2='332.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='703.3' y='332.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.9' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 38.8 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 38.8 s Datasets: 71'/><circle class='sa-hit' cx='338.2' cy='88.2' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 9.2% Selection time per 1k features: 0.626 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 9.2% Selection time per 1k features: 0.626 s Datasets: 71'/><circle class='sa-hit' cx='563.3' cy='114.1' r='10.5' data-tip='mRMR Improvement over Random: 8.3% Selection time per 1k features: 113 s Datasets: 71' aria-label='mRMR Improvement over Random: 8.3% Selection time per 1k features: 113 s Datasets: 71'/><circle class='sa-hit' cx='474.4' cy='129.0' r='10.5' data-tip='CART Improvement over Random: 7.7% Selection time per 1k features: 14.5 s Datasets: 71 Next to MarkovBlanket: 7.1%, 17.9 s' aria-label='CART Improvement over Random: 7.7% Selection time per 1k features: 14.5 s Datasets: 71 Next to MarkovBlanket: 7.1%, 17.9 s'/><circle class='sa-hit' cx='688.9' cy='134.3' r='10.5' data-tip='Lasso Improvement over Random: 7.6% Selection time per 1k features: 2.06e+03 s Datasets: 71' aria-label='Lasso Improvement over Random: 7.6% Selection time per 1k features: 2.06e+03 s Datasets: 71'/><circle class='sa-hit' cx='483.5' cy='146.4' r='10.5' data-tip='MarkovBlanket Improvement over Random: 7.1% Selection time per 1k features: 17.9 s Datasets: 71 Next to CART: 7.7%, 14.5 s' aria-label='MarkovBlanket Improvement over Random: 7.1% Selection time per 1k features: 17.9 s Datasets: 71 Next to CART: 7.7%, 14.5 s'/><circle class='sa-hit' cx='795.1' cy='149.4' r='10.5' data-tip='(R)ReliefF Improvement over Random: 7.0% Selection time per 1k features: 2.39e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 7.0% Selection time per 1k features: 2.39e+04 s Datasets: 71'/><circle class='sa-hit' cx='689.0' cy='169.3' r='10.5' data-tip='ElasticNet Improvement over Random: 6.3% Selection time per 1k features: 2.06e+03 s Datasets: 71' aria-label='ElasticNet Improvement over Random: 6.3% Selection time per 1k features: 2.06e+03 s Datasets: 71'/><circle class='sa-hit' cx='776.9' cy='199.6' r='10.5' data-tip='SBE Improvement over Random: 5.3% Selection time per 1k features: 1.57e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 5.3% Selection time per 1k features: 1.57e+04 s Datasets: 71'/><circle class='sa-hit' cx='468.0' cy='206.5' r='10.5' data-tip='GainRatio Improvement over Random: 5.1% Selection time per 1k features: 12.5 s Datasets: 71' aria-label='GainRatio Improvement over Random: 5.1% Selection time per 1k features: 12.5 s Datasets: 71'/><circle class='sa-hit' cx='524.8' cy='219.1' r='10.5' data-tip='MI Improvement over Random: 4.6% Selection time per 1k features: 46.5 s Datasets: 71' aria-label='MI Improvement over Random: 4.6% Selection time per 1k features: 46.5 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='237.0' r='10.5' data-tip='SFS Improvement over Random: 4.0% Selection time per 1k features: 3.56e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 4.0% Selection time per 1k features: 3.56e+04 s Datasets: 71'/><circle class='sa-hit' cx='737.3' cy='318.6' r='10.5' data-tip='LOCO Improvement over Random: 1.2% Selection time per 1k features: 6.29e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 1.2% Selection time per 1k features: 6.29e+03 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,11.0971,38.79,71,True
|
| 37 |
F-Test,9.1536,0.6257,71,True
|
| 38 |
mRMR,8.2563,113.024,71,False
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='159.1' x2='159.1' y1='16.2' y2='368.0'/><text class='sa-text' x='159.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='258.8' x2='258.8' y1='16.2' y2='368.0'/><text class='sa-text' x='258.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='358.5' x2='358.5' y1='16.2' y2='368.0'/><text class='sa-text' x='358.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='458.2' x2='458.2' y1='16.2' y2='368.0'/><text class='sa-text' x='458.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='558.0' x2='558.0' y1='16.2' y2='368.0'/><text class='sa-text' x='558.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.7' x2='657.7' y1='16.2' y2='368.0'/><text class='sa-text' x='657.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.4' x2='757.4' y1='16.2' y2='368.0'/><text class='sa-text' x='757.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='294.4' y2='294.4'/><text class='sa-text' x='57.9' y='300.4' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='236.8' y2='236.8'/><text class='sa-text' x='57.9' y='242.8' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='179.1' y2='179.1'/><text class='sa-text' x='57.9' y='185.1' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='121.5' y2='121.5'/><text class='sa-text' x='57.9' y='127.5' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='63.8' y2='63.8'/><text class='sa-text' x='57.9' y='69.8' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 338.2,88.2 516.9,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.9' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='338.2' cy='88.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='563.3' cy='114.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.4' cy='129.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='688.9' cy='134.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='483.5' cy='146.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.1' cy='149.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='689.0' cy='169.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='776.9' cy='199.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.0' cy='206.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='524.8' cy='219.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='237.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.3' cy='318.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.9' y1='32.2' x2='410.5' y2='46.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='410.5' y='46.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='338.2' y1='88.2' x2='298.6' y2='102.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='298.6' y='102.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='563.3' y1='114.1' x2='608.6' y2='123.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='608.6' y='123.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='474.4' y1='129.0' x2='495.8' y2='114.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='495.8' y='114.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='688.9' y1='134.3' x2='710.3' y2='104.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='710.3' y='104.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='483.5' y1='146.4' x2='590.0' y2='158.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='590.0' y='158.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><text class='sa-dim sa-halo' x='796.9' y='136.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><text class='sa-dim sa-halo' x='697.2' y='188.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='776.9' y1='199.6' x2='817.2' y2='196.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='817.2' y='196.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><text class='sa-dim sa-halo' x='458.6' y='191.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='543.1' y='220.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='812.3' y1='237.0' x2='828.6' y2='251.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='251.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='737.3' y1='318.6' x2='703.3' y2='332.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='703.3' y='332.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.9' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 38.8 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 38.8 s Datasets: 71'/><circle class='sa-hit' cx='338.2' cy='88.2' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 9.2% Selection time per 1k features: 0.626 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 9.2% Selection time per 1k features: 0.626 s Datasets: 71'/><circle class='sa-hit' cx='563.3' cy='114.1' r='10.5' data-tip='mRMR Improvement over Random: 8.3% Selection time per 1k features: 113 s Datasets: 71' aria-label='mRMR Improvement over Random: 8.3% Selection time per 1k features: 113 s Datasets: 71'/><circle class='sa-hit' cx='474.4' cy='129.0' r='10.5' data-tip='CART Improvement over Random: 7.7% Selection time per 1k features: 14.5 s Datasets: 71 Next to MarkovBlanket: 7.1%, 17.9 s' aria-label='CART Improvement over Random: 7.7% Selection time per 1k features: 14.5 s Datasets: 71 Next to MarkovBlanket: 7.1%, 17.9 s'/><circle class='sa-hit' cx='688.9' cy='134.3' r='10.5' data-tip='Lasso Improvement over Random: 7.6% Selection time per 1k features: 2.06e+03 s Datasets: 71' aria-label='Lasso Improvement over Random: 7.6% Selection time per 1k features: 2.06e+03 s Datasets: 71'/><circle class='sa-hit' cx='483.5' cy='146.4' r='10.5' data-tip='MarkovBlanket Improvement over Random: 7.1% Selection time per 1k features: 17.9 s Datasets: 71 Next to CART: 7.7%, 14.5 s' aria-label='MarkovBlanket Improvement over Random: 7.1% Selection time per 1k features: 17.9 s Datasets: 71 Next to CART: 7.7%, 14.5 s'/><circle class='sa-hit' cx='795.1' cy='149.4' r='10.5' data-tip='(R)ReliefF Improvement over Random: 7.0% Selection time per 1k features: 2.39e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 7.0% Selection time per 1k features: 2.39e+04 s Datasets: 71'/><circle class='sa-hit' cx='689.0' cy='169.3' r='10.5' data-tip='ElasticNet Improvement over Random: 6.3% Selection time per 1k features: 2.06e+03 s Datasets: 71' aria-label='ElasticNet Improvement over Random: 6.3% Selection time per 1k features: 2.06e+03 s Datasets: 71'/><circle class='sa-hit' cx='776.9' cy='199.6' r='10.5' data-tip='SBE Improvement over Random: 5.3% Selection time per 1k features: 1.57e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 5.3% Selection time per 1k features: 1.57e+04 s Datasets: 71'/><circle class='sa-hit' cx='468.0' cy='206.5' r='10.5' data-tip='GainRatio Improvement over Random: 5.1% Selection time per 1k features: 12.5 s Datasets: 71' aria-label='GainRatio Improvement over Random: 5.1% Selection time per 1k features: 12.5 s Datasets: 71'/><circle class='sa-hit' cx='524.8' cy='219.1' r='10.5' data-tip='MI Improvement over Random: 4.6% Selection time per 1k features: 46.5 s Datasets: 71' aria-label='MI Improvement over Random: 4.6% Selection time per 1k features: 46.5 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='237.0' r='10.5' data-tip='SFS Improvement over Random: 4.0% Selection time per 1k features: 3.56e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 4.0% Selection time per 1k features: 3.56e+04 s Datasets: 71'/><circle class='sa-hit' cx='737.3' cy='318.6' r='10.5' data-tip='LOCO Improvement over Random: 1.2% Selection time per 1k features: 6.29e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 1.2% Selection time per 1k features: 6.29e+03 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,11.0971,38.79,71,True
|
| 37 |
F-Test,9.1536,0.6257,71,True
|
| 38 |
mRMR,8.2563,113.024,71,False
|
data/subsets/tasks_all/models_lightgbm/budgets_4/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1274.0,1230.0,1326.0,5.9,65.0,8.7,,0.597,0.507,0.2,0.0,71,213
|
| 3 |
-
2,CART,Non-search,Tree,1177.0,1128.0,1234.0,6.9,57.9,11.0,,0.324,0.482,0.1,0.0,71,213
|
| 4 |
-
3,ElasticNet,Non-search,Regularization,1171.0,1122.0,1220.0,7.05,56.8,10.8,,0.308,0.568,6.1,0.0,71,213
|
| 5 |
-
4,Lasso,Non-search,Regularization,1169.0,1128.0,1215.0,7.22,55.6,10.5,,0.246,0.563,1.8,0.0,71,213
|
| 6 |
-
5,MI,Non-search,Information theory,1160.0,1119.0,1204.0,7.29,55.1,11.4,,0.587,0.627,0.4,0.0,71,213
|
| 7 |
-
6,F-Test,Non-search,Correlation,1154.0,1104.0,1199.0,7.5,53.6,10.2,,0.562,0.689,0.0,0.0,71,213
|
| 8 |
-
7,(R)ReliefF,Non-search,Distance,1148.0,1100.0,1193.0,7.86,51.0,11.2,,0.612,0.594,13.2,0.0,71,213
|
| 9 |
-
8,mRMR,Non-search,Information theory,1125.0,1077.0,1170.0,7.75,51.8,10.2,,0.651,0.53,0.5,9.9,71,213
|
| 10 |
-
9,MarkovBlanket,Search,Backward search,1116.0,1068.0,1162.0,8.39,47.2,11.3,,0.404,0.463,0.1,0.0,71,213
|
| 11 |
-
10,SBE,Search,Backward search,1112.0,1059.0,1163.0,8.4,47.1,13.1,,0.059,0.517,1001.8,40.8,71,213
|
| 12 |
-
11,SFS,Search,Forward search,1097.0,1048.0,1147.0,8.7,45.0,13.1,,0.176,0.525,3227.2,48.4,71,213
|
| 13 |
-
12,LOCO,Search,Leave-one-out,1092.0,1042.0,1145.0,8.36,47.4,13.8,,0.08,0.52,249.4,18.3,71,213
|
| 14 |
-
13,GainRatio,Non-search,Information theory,1091.0,1045.0,1132.0,8.61,45.7,12.7,,0.55,0.659,0.2,0.0,71,213
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,944.0,1053.0,9.8,37.1,16.9,,0.002,0.5,0.0,0.0,71,213
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,968.0,894.0,1024.0,10.28,33.7,20.2,,0.678,0.701,0.2,0.0,71,213
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1274.0,1230.0,1326.0,5.9,65.0,8.7,11.1,,0.597,0.507,0.2,0.0,71,213
|
| 3 |
+
2,CART,Non-search,Tree,1177.0,1128.0,1234.0,6.9,57.9,11.0,7.7,,0.324,0.482,0.1,0.0,71,213
|
| 4 |
+
3,ElasticNet,Non-search,Regularization,1171.0,1122.0,1220.0,7.05,56.8,10.8,6.3,,0.308,0.568,6.1,0.0,71,213
|
| 5 |
+
4,Lasso,Non-search,Regularization,1169.0,1128.0,1215.0,7.22,55.6,10.5,7.6,,0.246,0.563,1.8,0.0,71,213
|
| 6 |
+
5,MI,Non-search,Information theory,1160.0,1119.0,1204.0,7.29,55.1,11.4,4.6,,0.587,0.627,0.4,0.0,71,213
|
| 7 |
+
6,F-Test,Non-search,Correlation,1154.0,1104.0,1199.0,7.5,53.6,10.2,9.2,,0.562,0.689,0.0,0.0,71,213
|
| 8 |
+
7,(R)ReliefF,Non-search,Distance,1148.0,1100.0,1193.0,7.86,51.0,11.2,7.0,,0.612,0.594,13.2,0.0,71,213
|
| 9 |
+
8,mRMR,Non-search,Information theory,1125.0,1077.0,1170.0,7.75,51.8,10.2,8.3,,0.651,0.53,0.5,9.9,71,213
|
| 10 |
+
9,MarkovBlanket,Search,Backward search,1116.0,1068.0,1162.0,8.39,47.2,11.3,7.1,,0.404,0.463,0.1,0.0,71,213
|
| 11 |
+
10,SBE,Search,Backward search,1112.0,1059.0,1163.0,8.4,47.1,13.1,5.3,,0.059,0.517,1001.8,40.8,71,213
|
| 12 |
+
11,SFS,Search,Forward search,1097.0,1048.0,1147.0,8.7,45.0,13.1,4.0,,0.176,0.525,3227.2,48.4,71,213
|
| 13 |
+
12,LOCO,Search,Leave-one-out,1092.0,1042.0,1145.0,8.36,47.4,13.8,1.2,,0.08,0.52,249.4,18.3,71,213
|
| 14 |
+
13,GainRatio,Non-search,Information theory,1091.0,1045.0,1132.0,8.61,45.7,12.7,5.1,,0.55,0.659,0.2,0.0,71,213
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,944.0,1053.0,9.8,37.1,16.9,0.0,,0.002,0.5,0.0,0.0,71,213
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,968.0,894.0,1024.0,10.28,33.7,20.2,-220.2,,0.678,0.701,0.2,0.0,71,213
|
data/subsets/tasks_all/models_lightgbm/budgets_all/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1313.0'>1313<span class='sub'>+44 / −39</span></td><td class='num' data-sort='6.08'>6.08</td><td class='num' data-sort='13.4'>13.4</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1267.0'>1267<span class='sub'>+39 / −33</span></td><td class='num' data-sort='6.67'>6.67</td><td class='num' data-sort='16.1'>16.1</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+36 / −37</span></td><td class='num' data-sort='6.86'>6.86</td><td class='num' data-sort='16.4'>16.4</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1244.0'>1244<span class='sub'>+32 / −30</span></td><td class='num' data-sort='7.09'>7.09</td><td class='num' data-sort='16.6'>16.6</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1244.0'>1244<span class='sub'>+36 / −36</span></td><td class='num' data-sort='7.06'>7.06</td><td class='num' data-sort='16.4'>16.4</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1224.0'>1224<span class='sub'>+31 / −28</span></td><td class='num' data-sort='7.48'>7.48</td><td class='num' data-sort='17.5'>17.5</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1217.0'>1217<span class='sub'>+34 / −34</span></td><td class='num' data-sort='7.56'>7.56</td><td class='num' data-sort='17.3'>17.3</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1201.0'>1201<span class='sub'>+33 / −33</span></td><td class='num' data-sort='7.9'>7.90</td><td class='num' data-sort='17.8'>17.8</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+38 / −40</span></td><td class='num' data-sort='7.96'>7.96</td><td class='num' data-sort='19.5'>19.5</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2166.9'>2,166.9</td><td class='num' data-sort='43.8'>43.8</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1185.0'>1185<span class='sub'>+37 / −36</span></td><td class='num' data-sort='8.07'>8.07</td><td class='num' data-sort='18.9'>18.9</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='13.1'>13.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1184.0'>1184<span class='sub'>+44 / −46</span></td><td class='num' data-sort='8.09'>8.09</td><td class='num' data-sort='21.2'>21.2</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2611.6'>2,611.6</td><td class='num' data-sort='46.2'>46.2</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1181.0'>1181<span class='sub'>+32 / −33</span></td><td class='num' data-sort='8.25'>8.25</td><td class='num' data-sort='19.3'>19.3</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1158.0'>1158<span class='sub'>+41 / −40</span></td><td class='num' data-sort='8.35'>8.35</td><td class='num' data-sort='21.6'>21.6</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='240.9'>240.9</td><td class='num' data-sort='7.2'>7.2</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+36 / −38</span></td><td class='num' data-sort='10.78'>10.78</td><td class='num' data-sort='30.3'>30.3</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='943.0'>943<span class='sub'>+54 / −65</span></td><td class='num' data-sort='11.8'>11.80</td><td class='num' data-sort='35.1'>35.1</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1313.0,1274.0,1357.0,
|
| 37 |
-
2,CART,Non-search,Tree,1267.0,1234.0,1306.0,
|
| 38 |
-
3,ElasticNet,Non-search,Regularization,1252.0,1215.0,1288.0,
|
| 39 |
-
4,MI,Non-search,Information theory,1244.0,1214.0,1276.0,
|
| 40 |
-
5,Lasso,Non-search,Regularization,1244.0,1208.0,1280.0,
|
| 41 |
-
6,MarkovBlanket,Search,Backward search,1224.0,1196.0,1255.0,
|
| 42 |
-
7,F-Test,Non-search,Correlation,1217.0,1183.0,1251.0,
|
| 43 |
-
8,mRMR,Non-search,Information theory,1201.0,1168.0,1234.0,
|
| 44 |
-
9,SFS,Search,Forward search,1192.0,1152.0,1230.0,
|
| 45 |
-
10,(R)ReliefF,Non-search,Distance,1185.0,1149.0,1222.0,
|
| 46 |
-
11,SBE,Search,Backward search,1184.0,1138.0,1228.0,
|
| 47 |
-
12,GainRatio,Non-search,Information theory,1181.0,1148.0,1213.0,
|
| 48 |
-
13,LOCO,Search,Leave-one-out,1158.0,1118.0,1199.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,962.0,1036.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,943.0,878.0,997.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1313.0'>1313<span class='sub'>+44 / −39</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1267.0'>1267<span class='sub'>+39 / −33</span></td><td class='num' data-sort='23.5'>23.5</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1252.0'>1252<span class='sub'>+36 / −37</span></td><td class='num' data-sort='21.6'>21.6</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='6.1'>6.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1244.0'>1244<span class='sub'>+32 / −30</span></td><td class='num' data-sort='22.9'>22.9</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1244.0'>1244<span class='sub'>+36 / −36</span></td><td class='num' data-sort='21.1'>21.1</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1224.0'>1224<span class='sub'>+31 / −28</span></td><td class='num' data-sort='21.2'>21.2</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1217.0'>1217<span class='sub'>+34 / −34</span></td><td class='num' data-sort='21.5'>21.5</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1201.0'>1201<span class='sub'>+33 / −33</span></td><td class='num' data-sort='21.7'>21.7</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1192.0'>1192<span class='sub'>+38 / −40</span></td><td class='num' data-sort='18.1'>18.1</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2166.9'>2,166.9</td><td class='num' data-sort='43.8'>43.8</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1185.0'>1185<span class='sub'>+37 / −36</span></td><td class='num' data-sort='18.5'>18.5</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='13.1'>13.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1184.0'>1184<span class='sub'>+44 / −46</span></td><td class='num' data-sort='12.3'>12.3</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2611.6'>2,611.6</td><td class='num' data-sort='46.2'>46.2</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1181.0'>1181<span class='sub'>+32 / −33</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1158.0'>1158<span class='sub'>+41 / −40</span></td><td class='num' data-sort='13.0'>13.0</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='240.9'>240.9</td><td class='num' data-sort='7.2'>7.2</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+36 / −38</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='943.0'>943<span class='sub'>+54 / −65</span></td><td class='num' data-sort='-23.2'>-23.2</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1313.0,1274.0,1357.0,25.5,0.643,0.669,0.2,0.0
|
| 37 |
+
2,CART,Non-search,Tree,1267.0,1234.0,1306.0,23.5,0.422,0.586,0.0,0.0
|
| 38 |
+
3,ElasticNet,Non-search,Regularization,1252.0,1215.0,1288.0,21.6,0.447,0.76,6.1,0.0
|
| 39 |
+
4,MI,Non-search,Information theory,1244.0,1214.0,1276.0,22.9,0.647,0.785,0.4,0.0
|
| 40 |
+
5,Lasso,Non-search,Regularization,1244.0,1208.0,1280.0,21.1,0.362,0.73,1.9,0.0
|
| 41 |
+
6,MarkovBlanket,Search,Backward search,1224.0,1196.0,1255.0,21.2,0.413,0.463,0.1,0.0
|
| 42 |
+
7,F-Test,Non-search,Correlation,1217.0,1183.0,1251.0,21.5,0.679,0.846,0.0,0.0
|
| 43 |
+
8,mRMR,Non-search,Information theory,1201.0,1168.0,1234.0,21.7,0.604,0.655,0.4,3.4
|
| 44 |
+
9,SFS,Search,Forward search,1192.0,1152.0,1230.0,18.1,0.267,0.627,2166.9,43.8
|
| 45 |
+
10,(R)ReliefF,Non-search,Distance,1185.0,1149.0,1222.0,18.5,0.646,0.74,13.1,0.0
|
| 46 |
+
11,SBE,Search,Backward search,1184.0,1138.0,1228.0,12.3,0.132,0.56,2611.6,46.2
|
| 47 |
+
12,GainRatio,Non-search,Information theory,1181.0,1148.0,1213.0,19.8,0.644,0.825,0.2,0.0
|
| 48 |
+
13,LOCO,Search,Leave-one-out,1158.0,1118.0,1199.0,13.0,0.173,0.583,240.9,7.2
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,962.0,1036.0,0.0,0.001,0.5,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,943.0,878.0,997.0,-23.2,0.652,0.797,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_lightgbm/budgets_all/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='161.4' x2='161.4' y1='16.2' y2='368.0'/><text class='sa-text' x='161.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='262.2' x2='262.2' y1='16.2' y2='368.0'/><text class='sa-text' x='262.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.9' x2='362.9' y1='16.2' y2='368.0'/><text class='sa-text' x='362.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='463.7' x2='463.7' y1='16.2' y2='368.0'/><text class='sa-text' x='463.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='564.5' x2='564.5' y1='16.2' y2='368.0'/><text class='sa-text' x='564.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='665.3' x2='665.3' y1='16.2' y2='368.0'/><text class='sa-text' x='665.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='766.1' x2='766.1' y1='16.2' y2='368.0'/><text class='sa-text' x='766.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='289.4' y2='289.4'/><text class='sa-text' x='57.9' y='295.4' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='226.8' y2='226.8'/><text class='sa-text' x='57.9' y='232.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='164.2' y2='164.2'/><text class='sa-text' x='57.9' y='170.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='101.6' y2='101.6'/><text class='sa-text' x='57.9' y='107.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='39.0' y2='39.0'/><text class='sa-text' x='57.9' y='45.0' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 343.0,83.2 480.0,57.7 522.9,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='522.9' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='480.0' cy='57.7' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='530.8' cy='65.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='534.0' cy='81.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='698.0' cy='81.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='343.0' cy='83.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='491.2' cy='86.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='88.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.2' cy='104.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='804.2' cy='120.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='125.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='745.9' cy='189.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='810.3' cy='197.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='522.9' y1='32.2' x2='658.3' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='658.3' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='480.0' y1='57.7' x2='439.7' y2='72.0' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='439.7' y='72.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><text class='sa-dim sa-halo' x='549.0' y='54.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='534.0' y1='81.0' x2='578.7' y2='91.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='578.7' y='91.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='698.0' y1='81.7' x2='793.9' y2='77.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='793.9' y='77.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='343.0' y1='83.2' x2='302.7' y2='96.9' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='302.7' y='96.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='491.2' y1='86.9' x2='580.0' y2='117.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='580.0' y='117.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='695.4' y1='88.3' x2='664.9' y2='52.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='664.9' y='52.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='474.2' y1='104.5' x2='450.9' y2='157.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='450.9' y='157.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='789.4' y='109.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='125.7' x2='828.6' y2='139.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='139.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='745.9' y1='189.2' x2='769.2' y2='174.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='769.2' y='174.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='810.3' y1='197.4' x2='827.3' y2='211.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='827.3' y='211.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='522.9' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 25.5% Selection time per 1k features: 38.6 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 25.5% Selection time per 1k features: 38.6 s Datasets: 71'/><circle class='sa-hit' cx='480.0' cy='57.7' r='10.5' data-tip='CART (on the Pareto front) Improvement over Random: 23.5% Selection time per 1k features: 14.5 s Datasets: 71' aria-label='CART (on the Pareto front) Improvement over Random: 23.5% Selection time per 1k features: 14.5 s Datasets: 71'/><circle class='sa-hit' cx='530.8' cy='65.1' r='10.5' data-tip='MI Improvement over Random: 22.9% Selection time per 1k features: 46.3 s Datasets: 71 Next to mRMR: 21.7%, 49.9 s' aria-label='MI Improvement over Random: 22.9% Selection time per 1k features: 46.3 s Datasets: 71 Next to mRMR: 21.7%, 49.9 s'/><circle class='sa-hit' cx='534.0' cy='81.0' r='10.5' data-tip='mRMR Improvement over Random: 21.7% Selection time per 1k features: 49.9 s Datasets: 71 Next to MI: 22.9%, 46.3 s' aria-label='mRMR Improvement over Random: 21.7% Selection time per 1k features: 49.9 s Datasets: 71 Next to MI: 22.9%, 46.3 s'/><circle class='sa-hit' cx='698.0' cy='81.7' r='10.5' data-tip='ElasticNet Improvement over Random: 21.6% Selection time per 1k features: 2.11e+03 s Datasets: 71 Next to Lasso: 21.1%, 1.99e+03 s' aria-label='ElasticNet Improvement over Random: 21.6% Selection time per 1k features: 2.11e+03 s Datasets: 71 Next to Lasso: 21.1%, 1.99e+03 s'/><circle class='sa-hit' cx='343.0' cy='83.2' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 21.5% Selection time per 1k features: 0.635 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 21.5% Selection time per 1k features: 0.635 s Datasets: 71'/><circle class='sa-hit' cx='491.2' cy='86.9' r='10.5' data-tip='MarkovBlanket Improvement over Random: 21.2% Selection time per 1k features: 18.7 s Datasets: 71' aria-label='MarkovBlanket Improvement over Random: 21.2% Selection time per 1k features: 18.7 s Datasets: 71'/><circle class='sa-hit' cx='695.4' cy='88.3' r='10.5' data-tip='Lasso Improvement over Random: 21.1% Selection time per 1k features: 1.99e+03 s Datasets: 71 Next to ElasticNet: 21.6%, 2.11e+03 s' aria-label='Lasso Improvement over Random: 21.1% Selection time per 1k features: 1.99e+03 s Datasets: 71 Next to ElasticNet: 21.6%, 2.11e+03 s'/><circle class='sa-hit' cx='474.2' cy='104.5' r='10.5' data-tip='GainRatio Improvement over Random: 19.8% Selection time per 1k features: 12.7 s Datasets: 71' aria-label='GainRatio Improvement over Random: 19.8% Selection time per 1k features: 12.7 s Datasets: 71'/><circle class='sa-hit' cx='804.2' cy='120.6' r='10.5' data-tip='(R)ReliefF Improvement over Random: 18.5% Selection time per 1k features: 2.39e+04 s Datasets: 71 Next to SFS: 18.1%, 2.88e+04 s' aria-label='(R)ReliefF Improvement over Random: 18.5% Selection time per 1k features: 2.39e+04 s Datasets: 71 Next to SFS: 18.1%, 2.88e+04 s'/><circle class='sa-hit' cx='812.3' cy='125.7' r='10.5' data-tip='SFS Improvement over Random: 18.1% Selection time per 1k features: 2.88e+04 s Datasets: 71 Next to (R)ReliefF: 18.5%, 2.39e+04 s' aria-label='SFS Improvement over Random: 18.1% Selection time per 1k features: 2.88e+04 s Datasets: 71 Next to (R)ReliefF: 18.5%, 2.39e+04 s'/><circle class='sa-hit' cx='745.9' cy='189.2' r='10.5' data-tip='LOCO Improvement over Random: 13.0% Selection time per 1k features: 6.31e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 13.0% Selection time per 1k features: 6.31e+03 s Datasets: 71'/><circle class='sa-hit' cx='810.3' cy='197.4' r='10.5' data-tip='SBE Improvement over Random: 12.3% Selection time per 1k features: 2.75e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 12.3% Selection time per 1k features: 2.75e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,25.5467,38.6316,71,True
|
| 37 |
CART,23.5071,14.4928,71,True
|
| 38 |
MI,22.9183,46.2611,71,False
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='161.4' x2='161.4' y1='16.2' y2='368.0'/><text class='sa-text' x='161.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='262.2' x2='262.2' y1='16.2' y2='368.0'/><text class='sa-text' x='262.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.9' x2='362.9' y1='16.2' y2='368.0'/><text class='sa-text' x='362.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='463.7' x2='463.7' y1='16.2' y2='368.0'/><text class='sa-text' x='463.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='564.5' x2='564.5' y1='16.2' y2='368.0'/><text class='sa-text' x='564.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='665.3' x2='665.3' y1='16.2' y2='368.0'/><text class='sa-text' x='665.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='766.1' x2='766.1' y1='16.2' y2='368.0'/><text class='sa-text' x='766.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='289.4' y2='289.4'/><text class='sa-text' x='57.9' y='295.4' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='226.8' y2='226.8'/><text class='sa-text' x='57.9' y='232.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='164.2' y2='164.2'/><text class='sa-text' x='57.9' y='170.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='101.6' y2='101.6'/><text class='sa-text' x='57.9' y='107.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='39.0' y2='39.0'/><text class='sa-text' x='57.9' y='45.0' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 343.0,83.2 480.0,57.7 522.9,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='522.9' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle class='sa-point' cx='480.0' cy='57.7' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='530.8' cy='65.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='534.0' cy='81.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='698.0' cy='81.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='343.0' cy='83.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='491.2' cy='86.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='88.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.2' cy='104.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='804.2' cy='120.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='125.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='745.9' cy='189.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='810.3' cy='197.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='522.9' y1='32.2' x2='647.6' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='647.6' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='480.0' y1='57.7' x2='440.9' y2='71.4' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='440.9' y='71.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>CART</text><text class='sa-dim sa-halo' x='549.0' y='54.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='534.0' y1='81.0' x2='584.4' y2='91.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='584.4' y='91.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='698.0' y1='81.7' x2='793.2' y2='71.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='793.2' y='71.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='343.0' y1='83.2' x2='301.5' y2='97.4' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='301.5' y='97.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='491.2' y1='86.9' x2='575.0' y2='125.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='575.0' y='125.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='695.4' y1='88.3' x2='665.5' y2='55.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='665.5' y='55.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='474.2' y1='104.5' x2='454.7' y2='153.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='454.7' y='153.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='804.2' y1='120.6' x2='786.9' y2='105.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='786.9' y='105.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='125.7' x2='828.6' y2='140.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='140.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='745.9' y1='189.2' x2='769.2' y2='174.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='769.2' y='174.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='810.3' y1='197.4' x2='827.3' y2='211.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='827.3' y='211.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='522.9' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 25.5% Selection time per 1k features: 38.6 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 25.5% Selection time per 1k features: 38.6 s Datasets: 71'/><circle class='sa-hit' cx='480.0' cy='57.7' r='10.5' data-tip='CART (on the Pareto front) Improvement over Random: 23.5% Selection time per 1k features: 14.5 s Datasets: 71' aria-label='CART (on the Pareto front) Improvement over Random: 23.5% Selection time per 1k features: 14.5 s Datasets: 71'/><circle class='sa-hit' cx='530.8' cy='65.1' r='10.5' data-tip='MI Improvement over Random: 22.9% Selection time per 1k features: 46.3 s Datasets: 71 Next to mRMR: 21.7%, 49.9 s' aria-label='MI Improvement over Random: 22.9% Selection time per 1k features: 46.3 s Datasets: 71 Next to mRMR: 21.7%, 49.9 s'/><circle class='sa-hit' cx='534.0' cy='81.0' r='10.5' data-tip='mRMR Improvement over Random: 21.7% Selection time per 1k features: 49.9 s Datasets: 71 Next to MI: 22.9%, 46.3 s' aria-label='mRMR Improvement over Random: 21.7% Selection time per 1k features: 49.9 s Datasets: 71 Next to MI: 22.9%, 46.3 s'/><circle class='sa-hit' cx='698.0' cy='81.7' r='10.5' data-tip='ElasticNet Improvement over Random: 21.6% Selection time per 1k features: 2.11e+03 s Datasets: 71 Next to Lasso: 21.1%, 1.99e+03 s' aria-label='ElasticNet Improvement over Random: 21.6% Selection time per 1k features: 2.11e+03 s Datasets: 71 Next to Lasso: 21.1%, 1.99e+03 s'/><circle class='sa-hit' cx='343.0' cy='83.2' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 21.5% Selection time per 1k features: 0.635 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 21.5% Selection time per 1k features: 0.635 s Datasets: 71'/><circle class='sa-hit' cx='491.2' cy='86.9' r='10.5' data-tip='MarkovBlanket Improvement over Random: 21.2% Selection time per 1k features: 18.7 s Datasets: 71' aria-label='MarkovBlanket Improvement over Random: 21.2% Selection time per 1k features: 18.7 s Datasets: 71'/><circle class='sa-hit' cx='695.4' cy='88.3' r='10.5' data-tip='Lasso Improvement over Random: 21.1% Selection time per 1k features: 1.99e+03 s Datasets: 71 Next to ElasticNet: 21.6%, 2.11e+03 s' aria-label='Lasso Improvement over Random: 21.1% Selection time per 1k features: 1.99e+03 s Datasets: 71 Next to ElasticNet: 21.6%, 2.11e+03 s'/><circle class='sa-hit' cx='474.2' cy='104.5' r='10.5' data-tip='GainRatio Improvement over Random: 19.8% Selection time per 1k features: 12.7 s Datasets: 71' aria-label='GainRatio Improvement over Random: 19.8% Selection time per 1k features: 12.7 s Datasets: 71'/><circle class='sa-hit' cx='804.2' cy='120.6' r='10.5' data-tip='(R)ReliefF Improvement over Random: 18.5% Selection time per 1k features: 2.39e+04 s Datasets: 71 Next to SFS: 18.1%, 2.88e+04 s' aria-label='(R)ReliefF Improvement over Random: 18.5% Selection time per 1k features: 2.39e+04 s Datasets: 71 Next to SFS: 18.1%, 2.88e+04 s'/><circle class='sa-hit' cx='812.3' cy='125.7' r='10.5' data-tip='SFS Improvement over Random: 18.1% Selection time per 1k features: 2.88e+04 s Datasets: 71 Next to (R)ReliefF: 18.5%, 2.39e+04 s' aria-label='SFS Improvement over Random: 18.1% Selection time per 1k features: 2.88e+04 s Datasets: 71 Next to (R)ReliefF: 18.5%, 2.39e+04 s'/><circle class='sa-hit' cx='745.9' cy='189.2' r='10.5' data-tip='LOCO Improvement over Random: 13.0% Selection time per 1k features: 6.31e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 13.0% Selection time per 1k features: 6.31e+03 s Datasets: 71'/><circle class='sa-hit' cx='810.3' cy='197.4' r='10.5' data-tip='SBE Improvement over Random: 12.3% Selection time per 1k features: 2.75e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 12.3% Selection time per 1k features: 2.75e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,25.5467,38.6316,71,True
|
| 37 |
CART,23.5071,14.4928,71,True
|
| 38 |
MI,22.9183,46.2611,71,False
|
data/subsets/tasks_all/models_lightgbm/budgets_all/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1313.0,1274.0,1357.0,6.08,63.7,13.4,,0.643,0.669,0.2,0.0,71,1065
|
| 3 |
-
2,CART,Non-search,Tree,1267.0,1234.0,1306.0,6.67,59.5,16.1,,0.422,0.586,0.0,0.0,71,1065
|
| 4 |
-
3,ElasticNet,Non-search,Regularization,1252.0,1215.0,1288.0,6.86,58.2,16.4,,0.447,0.76,6.1,0.0,71,1065
|
| 5 |
-
4,MI,Non-search,Information theory,1244.0,1214.0,1276.0,7.09,56.5,16.6,,0.647,0.785,0.4,0.0,71,1065
|
| 6 |
-
5,Lasso,Non-search,Regularization,1244.0,1208.0,1280.0,7.06,56.7,16.4,,0.362,0.73,1.9,0.0,71,1065
|
| 7 |
-
6,MarkovBlanket,Search,Backward search,1224.0,1196.0,1255.0,7.48,53.7,17.5,,0.413,0.463,0.1,0.0,71,1065
|
| 8 |
-
7,F-Test,Non-search,Correlation,1217.0,1183.0,1251.0,7.56,53.2,17.3,,0.679,0.846,0.0,0.0,71,1065
|
| 9 |
-
8,mRMR,Non-search,Information theory,1201.0,1168.0,1234.0,7.9,50.7,17.8,,0.604,0.655,0.4,3.4,71,1065
|
| 10 |
-
9,SFS,Search,Forward search,1192.0,1152.0,1230.0,7.96,50.3,19.5,,0.267,0.627,2166.9,43.8,71,1065
|
| 11 |
-
10,(R)ReliefF,Non-search,Distance,1185.0,1149.0,1222.0,8.07,49.5,18.9,,0.646,0.74,13.1,0.0,71,1065
|
| 12 |
-
11,SBE,Search,Backward search,1184.0,1138.0,1228.0,8.09,49.4,21.2,,0.132,0.56,2611.6,46.2,71,1065
|
| 13 |
-
12,GainRatio,Non-search,Information theory,1181.0,1148.0,1213.0,8.25,48.2,19.3,,0.644,0.825,0.2,0.0,71,1065
|
| 14 |
-
13,LOCO,Search,Leave-one-out,1158.0,1118.0,1199.0,8.35,47.5,21.6,,0.173,0.583,240.9,7.2,71,1065
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,962.0,1036.0,10.78,30.1,30.3,,0.001,0.5,0.0,0.0,71,1065
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,943.0,878.0,997.0,11.8,22.8,35.1,,0.652,0.797,0.2,0.0,71,1065
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1313.0,1274.0,1357.0,6.08,63.7,13.4,25.5,,0.643,0.669,0.2,0.0,71,1065
|
| 3 |
+
2,CART,Non-search,Tree,1267.0,1234.0,1306.0,6.67,59.5,16.1,23.5,,0.422,0.586,0.0,0.0,71,1065
|
| 4 |
+
3,ElasticNet,Non-search,Regularization,1252.0,1215.0,1288.0,6.86,58.2,16.4,21.6,,0.447,0.76,6.1,0.0,71,1065
|
| 5 |
+
4,MI,Non-search,Information theory,1244.0,1214.0,1276.0,7.09,56.5,16.6,22.9,,0.647,0.785,0.4,0.0,71,1065
|
| 6 |
+
5,Lasso,Non-search,Regularization,1244.0,1208.0,1280.0,7.06,56.7,16.4,21.1,,0.362,0.73,1.9,0.0,71,1065
|
| 7 |
+
6,MarkovBlanket,Search,Backward search,1224.0,1196.0,1255.0,7.48,53.7,17.5,21.2,,0.413,0.463,0.1,0.0,71,1065
|
| 8 |
+
7,F-Test,Non-search,Correlation,1217.0,1183.0,1251.0,7.56,53.2,17.3,21.5,,0.679,0.846,0.0,0.0,71,1065
|
| 9 |
+
8,mRMR,Non-search,Information theory,1201.0,1168.0,1234.0,7.9,50.7,17.8,21.7,,0.604,0.655,0.4,3.4,71,1065
|
| 10 |
+
9,SFS,Search,Forward search,1192.0,1152.0,1230.0,7.96,50.3,19.5,18.1,,0.267,0.627,2166.9,43.8,71,1065
|
| 11 |
+
10,(R)ReliefF,Non-search,Distance,1185.0,1149.0,1222.0,8.07,49.5,18.9,18.5,,0.646,0.74,13.1,0.0,71,1065
|
| 12 |
+
11,SBE,Search,Backward search,1184.0,1138.0,1228.0,8.09,49.4,21.2,12.3,,0.132,0.56,2611.6,46.2,71,1065
|
| 13 |
+
12,GainRatio,Non-search,Information theory,1181.0,1148.0,1213.0,8.25,48.2,19.3,19.8,,0.644,0.825,0.2,0.0,71,1065
|
| 14 |
+
13,LOCO,Search,Leave-one-out,1158.0,1118.0,1199.0,8.35,47.5,21.6,13.0,,0.173,0.583,240.9,7.2,71,1065
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,962.0,1036.0,10.78,30.1,30.3,0.0,,0.001,0.5,0.0,0.0,71,1065
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,943.0,878.0,997.0,11.8,22.8,35.1,-23.2,,0.652,0.797,0.2,0.0,71,1065
|
data/subsets/tasks_all/models_linearmodel/budgets_0/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1352.0'>1352<span class='sub'>+61 / −49</span></td><td class='num' data-sort='6.54'>6.54</td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1350.0'>1350<span class='sub'>+48 / −44</span></td><td class='num' data-sort='6.6'>6.60</td><td class='num' data-sort='19.5'>19.5</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1344.0'>1344<span class='sub'>+62 / −56</span></td><td class='num' data-sort='6.66'>6.66</td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1327.0'>1327<span class='sub'>+54 / −48</span></td><td class='num' data-sort='6.35'>6.35</td><td class='num' data-sort='20.5'>20.5</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='6.0'>6.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1324.0'>1324<span class='sub'>+44 / −43</span></td><td class='num' data-sort='6.75'>6.75</td><td class='num' data-sort='20.7'>20.7</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1323.0'>1323<span class='sub'>+44 / −41</span></td><td class='num' data-sort='6.9'>6.90</td><td class='num' data-sort='20.1'>20.1</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1301.0'>1301<span class='sub'>+45 / −48</span></td><td class='num' data-sort='7.44'>7.44</td><td class='num' data-sort='20.4'>20.4</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1295.0'>1295<span class='sub'>+51 / −52</span></td><td class='num' data-sort='7.39'>7.39</td><td class='num' data-sort='22.7'>22.7</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='970.7'>970.7</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1291.0'>1291<span class='sub'>+49 / −47</span></td><td class='num' data-sort='7.02'>7.02</td><td class='num' data-sort='22.1'>22.1</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+44 / −41</span></td><td class='num' data-sort='7.81'>7.81</td><td class='num' data-sort='24.0'>24.0</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1226.0'>1226<span class='sub'>+56 / −57</span></td><td class='num' data-sort='8.6'>8.60</td><td class='num' data-sort='25.1'>25.1</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='13.7'>13.7</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+45 / −45</span></td><td class='num' data-sort='8.73'>8.73</td><td class='num' data-sort='27.7'>27.7</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='245.6'>245.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+51 / −57</span></td><td class='num' data-sort='9.36'>9.36</td><td class='num' data-sort='30.0'>30.0</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3600.1'>3,600.1</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+54 / −62</span></td><td class='num' data-sort='11.53'>11.53</td><td class='num' data-sort='40.2'>40.2</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='973.0'>973<span class='sub'>+68 / −83</span></td><td class='num' data-sort='12.31'>12.31</td><td class='num' data-sort='42.8'>42.8</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1352.0,1303.0,1413.0,
|
| 37 |
-
2,F-Test,Non-search,Correlation,1350.0,1306.0,1398.0,
|
| 38 |
-
3,CART,Non-search,Tree,1344.0,1288.0,1406.0,
|
| 39 |
-
4,ElasticNet,Non-search,Regularization,1327.0,1279.0,1381.0,
|
| 40 |
-
5,MI,Non-search,Information theory,1324.0,1281.0,1368.0,
|
| 41 |
-
6,MarkovBlanket,Search,Backward search,1323.0,1282.0,1367.0,
|
| 42 |
-
7,mRMR,Non-search,Information theory,1301.0,1253.0,1346.0,
|
| 43 |
-
8,SFS,Search,Forward search,1295.0,1243.0,1346.0,
|
| 44 |
-
9,Lasso,Non-search,Regularization,1291.0,1244.0,1340.0,
|
| 45 |
-
10,GainRatio,Non-search,Information theory,1274.0,1233.0,1318.0,
|
| 46 |
-
11,(R)ReliefF,Non-search,Distance,1226.0,1169.0,1282.0,
|
| 47 |
-
12,LOCO,Search,Leave-one-out,1197.0,1152.0,1242.0,
|
| 48 |
-
13,SBE,Search,Backward search,1168.0,1111.0,1219.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,938.0,1054.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,973.0,890.0,1041.0,12.
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1352.0'>1352<span class='sub'>+61 / −49</span></td><td class='num' data-sort='28.3'>28.3</td><td class='num shaded' data-sort='0.628' style='background:rgba(0,114,178,0.50)'>0.628</td><td class='num shaded' data-sort='0.805' style='background:rgba(213,94,0,0.62)'>0.805</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1350.0'>1350<span class='sub'>+48 / −44</span></td><td class='num' data-sort='28.8'>28.8</td><td class='num shaded' data-sort='0.725' style='background:rgba(0,114,178,0.57)'>0.725</td><td class='num shaded' data-sort='0.952' style='background:rgba(213,94,0,0.72)'>0.952</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1344.0'>1344<span class='sub'>+62 / −56</span></td><td class='num' data-sort='27.0'>27.0</td><td class='num shaded' data-sort='0.484' style='background:rgba(0,114,178,0.40)'>0.484</td><td class='num shaded' data-sort='0.731' style='background:rgba(213,94,0,0.57)'>0.731</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1327.0'>1327<span class='sub'>+54 / −48</span></td><td class='num' data-sort='26.2'>26.2</td><td class='num shaded' data-sort='0.509' style='background:rgba(0,114,178,0.42)'>0.509</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='6.0'>6.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1324.0'>1324<span class='sub'>+44 / −43</span></td><td class='num' data-sort='28.0'>28.0</td><td class='num shaded' data-sort='0.645' style='background:rgba(0,114,178,0.51)'>0.645</td><td class='num shaded' data-sort='0.898' style='background:rgba(213,94,0,0.68)'>0.898</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1323.0'>1323<span class='sub'>+44 / −41</span></td><td class='num' data-sort='25.4'>25.4</td><td class='num shaded' data-sort='0.418' style='background:rgba(0,114,178,0.36)'>0.418</td><td class='num shaded' data-sort='0.465' style='background:rgba(213,94,0,0.39)'>0.465</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1301.0'>1301<span class='sub'>+45 / −48</span></td><td class='num' data-sort='26.3'>26.3</td><td class='num shaded' data-sort='0.528' style='background:rgba(0,114,178,0.43)'>0.528</td><td class='num shaded' data-sort='0.779' style='background:rgba(213,94,0,0.60)'>0.779</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1295.0'>1295<span class='sub'>+51 / −52</span></td><td class='num' data-sort='23.3'>23.3</td><td class='num shaded' data-sort='0.368' style='background:rgba(0,114,178,0.33)'>0.368</td><td class='num shaded' data-sort='0.75' style='background:rgba(213,94,0,0.58)'>0.750</td><td class='num' data-sort='970.7'>970.7</td><td class='num' data-sort='29.1'>29.1</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1291.0'>1291<span class='sub'>+49 / −47</span></td><td class='num' data-sort='24.5'>24.5</td><td class='num shaded' data-sort='0.421' style='background:rgba(0,114,178,0.36)'>0.421</td><td class='num shaded' data-sort='0.89' style='background:rgba(213,94,0,0.68)'>0.890</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+44 / −41</span></td><td class='num' data-sort='22.5'>22.5</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.942' style='background:rgba(213,94,0,0.71)'>0.942</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1226.0'>1226<span class='sub'>+56 / −57</span></td><td class='num' data-sort='18.0'>18.0</td><td class='num shaded' data-sort='0.642' style='background:rgba(0,114,178,0.51)'>0.642</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='13.7'>13.7</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1197.0'>1197<span class='sub'>+45 / −45</span></td><td class='num' data-sort='15.3'>15.3</td><td class='num shaded' data-sort='0.299' style='background:rgba(0,114,178,0.28)'>0.299</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='245.6'>245.6</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+51 / −57</span></td><td class='num' data-sort='8.8'>8.8</td><td class='num shaded' data-sort='0.195' style='background:rgba(0,114,178,0.21)'>0.195</td><td class='num shaded' data-sort='0.621' style='background:rgba(213,94,0,0.50)'>0.621</td><td class='num' data-sort='3600.1'>3,600.1</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+54 / −62</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.493' style='background:rgba(213,94,0,0.41)'>0.493</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='973.0'>973<span class='sub'>+68 / −83</span></td><td class='num' data-sort='-12.4'>-12.4</td><td class='num shaded' data-sort='0.588' style='background:rgba(0,114,178,0.47)'>0.588</td><td class='num shaded' data-sort='0.847' style='background:rgba(213,94,0,0.65)'>0.847</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1352.0,1303.0,1413.0,28.3,0.628,0.805,0.2,0.0
|
| 37 |
+
2,F-Test,Non-search,Correlation,1350.0,1306.0,1398.0,28.8,0.725,0.952,0.0,0.0
|
| 38 |
+
3,CART,Non-search,Tree,1344.0,1288.0,1406.0,27.0,0.484,0.731,0.0,0.0
|
| 39 |
+
4,ElasticNet,Non-search,Regularization,1327.0,1279.0,1381.0,26.2,0.509,0.915,6.0,0.0
|
| 40 |
+
5,MI,Non-search,Information theory,1324.0,1281.0,1368.0,28.0,0.645,0.898,0.4,0.0
|
| 41 |
+
6,MarkovBlanket,Search,Backward search,1323.0,1282.0,1367.0,25.4,0.418,0.465,0.1,0.0
|
| 42 |
+
7,mRMR,Non-search,Information theory,1301.0,1253.0,1346.0,26.3,0.528,0.779,0.3,0.0
|
| 43 |
+
8,SFS,Search,Forward search,1295.0,1243.0,1346.0,23.3,0.368,0.75,970.7,29.1
|
| 44 |
+
9,Lasso,Non-search,Regularization,1291.0,1244.0,1340.0,24.5,0.421,0.89,1.9,0.0
|
| 45 |
+
10,GainRatio,Non-search,Information theory,1274.0,1233.0,1318.0,22.5,0.679,0.942,0.2,0.0
|
| 46 |
+
11,(R)ReliefF,Non-search,Distance,1226.0,1169.0,1282.0,18.0,0.642,0.873,13.7,0.0
|
| 47 |
+
12,LOCO,Search,Leave-one-out,1197.0,1152.0,1242.0,15.3,0.299,0.665,245.6,0.0
|
| 48 |
+
13,SBE,Search,Backward search,1168.0,1111.0,1219.0,8.8,0.195,0.621,3600.1,49.3
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,938.0,1054.0,0.0,-0.001,0.493,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,973.0,890.0,1041.0,-12.4,0.588,0.847,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_linearmodel/budgets_0/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='162.3' x2='162.3' y1='20.0' y2='368.0'/><text class='sa-text' x='162.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='261.6' x2='261.6' y1='20.0' y2='368.0'/><text class='sa-text' x='261.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='360.9' x2='360.9' y1='20.0' y2='368.0'/><text class='sa-text' x='360.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='460.3' x2='460.3' y1='20.0' y2='368.0'/><text class='sa-text' x='460.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.6' x2='559.6' y1='20.0' y2='368.0'/><text class='sa-text' x='559.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='658.9' x2='658.9' y1='20.0' y2='368.0'/><text class='sa-text' x='658.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='758.3' x2='758.3' y1='20.0' y2='368.0'/><text class='sa-text' x='758.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.2' y2='352.2'/><text class='sa-text' x='58.0' y='358.2' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='297.2' y2='297.2'/><text class='sa-text' x='58.0' y='303.2' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='242.2' y2='242.2'/><text class='sa-text' x='58.0' y='248.2' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='187.2' y2='187.2'/><text class='sa-text' x='58.0' y='193.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='132.2' y2='132.2'/><text class='sa-text' x='58.0' y='138.2' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='77.2' y2='77.2'/><text class='sa-text' x='58.0' y='83.2' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='22.2' y2='22.2'/><text class='sa-text' x='58.0' y='28.2' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='20.0' width='780.8' height='348.1'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 194.0) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='55.3' x2='69.0' y2='22.0' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='79.3' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.2 341.0,35.8' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='341.0' cy='35.8' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='518.2' cy='41.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='526.6' cy='44.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='476.4' cy='55.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='481.0' cy='62.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.8' cy='64.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='487.9' cy='72.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='688.2' cy='82.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='782.3' cy='96.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='471.1' cy='104.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.8' cy='154.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='738.2' cy='184.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='255.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.2' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='341.0' y1='35.8' x2='263.5' y2='31.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='263.5' y='31.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='518.2' y1='41.0' x2='422.5' y2='29.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='422.5' y='29.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>RFImportance</text><line class='sa-ink' x1='526.6' y1='44.3' x2='425.9' y2='29.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='425.9' y='29.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='476.4' y1='55.5' x2='437.3' y2='62.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='437.3' y='62.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='481.0' y1='62.6' x2='402.2' y2='97.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='402.2' y='97.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='691.8' y1='64.5' x2='738.4' y2='49.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='738.4' y='49.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='487.9' y1='72.6' x2='579.2' y2='93.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='579.2' y='93.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='688.2' y1='82.6' x2='742.7' y2='109.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='742.7' y='109.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='782.3' y1='96.1' x2='808.4' y2='81.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='808.4' y='81.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='471.1' y1='104.4' x2='528.4' y2='128.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='528.4' y='128.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='797.0' y='139.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='738.2' y1='184.3' x2='705.5' y2='197.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='705.5' y='197.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='812.3' y1='255.6' x2='787.1' y2='269.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='787.1' y='269.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.2' x2='156.2' y2='338.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='338.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='341.0' cy='35.8' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 28.8% Selection time per 1k features: 0.629 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 28.8% Selection time per 1k features: 0.629 s Datasets: 71'/><circle class='sa-hit' cx='518.2' cy='41.0' r='10.5' data-tip='RFImportance Improvement over Random: 28.3% Selection time per 1k features: 38.3 s Datasets: 71 Next to MI: 28.0%, 46.6 s' aria-label='RFImportance Improvement over Random: 28.3% Selection time per 1k features: 38.3 s Datasets: 71 Next to MI: 28.0%, 46.6 s'/><circle class='sa-hit' cx='526.6' cy='44.3' r='10.5' data-tip='MI Improvement over Random: 28.0% Selection time per 1k features: 46.6 s Datasets: 71 Next to RFImportance: 28.3%, 38.3 s' aria-label='MI Improvement over Random: 28.0% Selection time per 1k features: 46.6 s Datasets: 71 Next to RFImportance: 28.3%, 38.3 s'/><circle class='sa-hit' cx='476.4' cy='55.5' r='10.5' data-tip='CART Improvement over Random: 27.0% Selection time per 1k features: 14.5 s Datasets: 71 Next to mRMR: 26.3%, 16.2 s Next to MarkovBlanket: 25.4%, 19 s' aria-label='CART Improvement over Random: 27.0% Selection time per 1k features: 14.5 s Datasets: 71 Next to mRMR: 26.3%, 16.2 s Next to MarkovBlanket: 25.4%, 19 s'/><circle class='sa-hit' cx='481.0' cy='62.6' r='10.5' data-tip='mRMR Improvement over Random: 26.3% Selection time per 1k features: 16.2 s Datasets: 71 Next to CART: 27.0%, 14.5 s Next to MarkovBlanket: 25.4%, 19 s' aria-label='mRMR Improvement over Random: 26.3% Selection time per 1k features: 16.2 s Datasets: 71 Next to CART: 27.0%, 14.5 s Next to MarkovBlanket: 25.4%, 19 s'/><circle class='sa-hit' cx='691.8' cy='64.5' r='10.5' data-tip='ElasticNet Improvement over Random: 26.2% Selection time per 1k features: 2.14e+03 s Datasets: 71 Next to Lasso: 24.5%, 1.97e+03 s' aria-label='ElasticNet Improvement over Random: 26.2% Selection time per 1k features: 2.14e+03 s Datasets: 71 Next to Lasso: 24.5%, 1.97e+03 s'/><circle class='sa-hit' cx='487.9' cy='72.6' r='10.5' data-tip='MarkovBlanket Improvement over Random: 25.4% Selection time per 1k features: 19 s Datasets: 71 Next to CART: 27.0%, 14.5 s Next to mRMR: 26.3%, 16.2 s' aria-label='MarkovBlanket Improvement over Random: 25.4% Selection time per 1k features: 19 s Datasets: 71 Next to CART: 27.0%, 14.5 s Next to mRMR: 26.3%, 16.2 s'/><circle class='sa-hit' cx='688.2' cy='82.6' r='10.5' data-tip='Lasso Improvement over Random: 24.5% Selection time per 1k features: 1.97e+03 s Datasets: 71 Next to ElasticNet: 26.2%, 2.14e+03 s' aria-label='Lasso Improvement over Random: 24.5% Selection time per 1k features: 1.97e+03 s Datasets: 71 Next to ElasticNet: 26.2%, 2.14e+03 s'/><circle class='sa-hit' cx='782.3' cy='96.1' r='10.5' data-tip='SFS Improvement over Random: 23.3% Selection time per 1k features: 1.75e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 23.3% Selection time per 1k features: 1.75e+04 s Datasets: 71'/><circle class='sa-hit' cx='471.1' cy='104.4' r='10.5' data-tip='GainRatio Improvement over Random: 22.5% Selection time per 1k features: 12.8 s Datasets: 71' aria-label='GainRatio Improvement over Random: 22.5% Selection time per 1k features: 12.8 s Datasets: 71'/><circle class='sa-hit' cx='795.8' cy='154.0' r='10.5' data-tip='(R)ReliefF Improvement over Random: 18.0% Selection time per 1k features: 2.39e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 18.0% Selection time per 1k features: 2.39e+04 s Datasets: 71'/><circle class='sa-hit' cx='738.2' cy='184.3' r='10.5' data-tip='LOCO Improvement over Random: 15.3% Selection time per 1k features: 6.29e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 15.3% Selection time per 1k features: 6.29e+03 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='255.6' r='10.5' data-tip='SBE Improvement over Random: 8.8% Selection time per 1k features: 3.5e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 8.8% Selection time per 1k features: 3.5e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.2' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0025 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0025 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
F-Test,28.7686,0.6293,71,True
|
| 37 |
RFImportance,28.295,38.3008,71,False
|
| 38 |
MI,27.9958,46.5786,71,False
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='162.3' x2='162.3' y1='20.0' y2='368.0'/><text class='sa-text' x='162.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='261.6' x2='261.6' y1='20.0' y2='368.0'/><text class='sa-text' x='261.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='360.9' x2='360.9' y1='20.0' y2='368.0'/><text class='sa-text' x='360.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='460.3' x2='460.3' y1='20.0' y2='368.0'/><text class='sa-text' x='460.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.6' x2='559.6' y1='20.0' y2='368.0'/><text class='sa-text' x='559.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='658.9' x2='658.9' y1='20.0' y2='368.0'/><text class='sa-text' x='658.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='758.3' x2='758.3' y1='20.0' y2='368.0'/><text class='sa-text' x='758.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='67.0' x2='847.8' y1='352.2' y2='352.2'/><text class='sa-text' x='58.0' y='358.2' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='67.0' x2='847.8' y1='297.2' y2='297.2'/><text class='sa-text' x='58.0' y='303.2' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='67.0' x2='847.8' y1='242.2' y2='242.2'/><text class='sa-text' x='58.0' y='248.2' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='67.0' x2='847.8' y1='187.2' y2='187.2'/><text class='sa-text' x='58.0' y='193.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='67.0' x2='847.8' y1='132.2' y2='132.2'/><text class='sa-text' x='58.0' y='138.2' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='67.0' x2='847.8' y1='77.2' y2='77.2'/><text class='sa-text' x='58.0' y='83.2' text-anchor='end' font-size='15'>25</text><line class='sa-grid' x1='67.0' x2='847.8' y1='22.2' y2='22.2'/><text class='sa-text' x='58.0' y='28.2' text-anchor='end' font-size='15'>30</text><rect class='sa-frame-line' x='67.0' y='20.0' width='780.8' height='348.1'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 194.0) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.6' y1='55.3' x2='69.0' y2='22.0' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.6' y='79.3' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.5,352.2 341.0,35.8' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='341.0' cy='35.8' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='518.2' cy='41.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='526.6' cy='44.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='476.4' cy='55.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='481.0' cy='62.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.8' cy='64.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='487.9' cy='72.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='688.2' cy='82.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='782.3' cy='96.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='471.1' cy='104.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.8' cy='154.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='738.2' cy='184.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='255.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.5' cy='352.2' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='341.0' y1='35.8' x2='258.5' y2='31.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='258.5' y='31.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='518.2' y1='41.0' x2='422.5' y2='29.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='422.5' y='29.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>RFImportance</text><line class='sa-ink' x1='526.6' y1='44.3' x2='425.9' y2='29.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='425.9' y='29.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='476.4' y1='55.5' x2='422.2' y2='62.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='422.2' y='62.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='481.0' y1='62.6' x2='402.2' y2='90.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='402.2' y='90.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='691.8' y1='64.5' x2='738.4' y2='49.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='738.4' y='49.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='487.9' y1='72.6' x2='579.8' y2='94.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='579.8' y='94.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='688.2' y1='82.6' x2='747.1' y2='113.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='747.1' y='113.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='782.3' y1='96.1' x2='814.1' y2='81.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='814.1' y='81.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='471.1' y1='104.4' x2='530.9' y2='129.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='530.9' y='129.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='797.0' y='143.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='738.2' y1='184.3' x2='705.5' y2='197.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='705.5' y='197.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='812.3' y1='255.6' x2='787.1' y2='269.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='787.1' y='269.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.5' y1='352.2' x2='156.2' y2='338.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='338.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='341.0' cy='35.8' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 28.8% Selection time per 1k features: 0.629 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 28.8% Selection time per 1k features: 0.629 s Datasets: 71'/><circle class='sa-hit' cx='518.2' cy='41.0' r='10.5' data-tip='RFImportance Improvement over Random: 28.3% Selection time per 1k features: 38.3 s Datasets: 71 Next to MI: 28.0%, 46.6 s' aria-label='RFImportance Improvement over Random: 28.3% Selection time per 1k features: 38.3 s Datasets: 71 Next to MI: 28.0%, 46.6 s'/><circle class='sa-hit' cx='526.6' cy='44.3' r='10.5' data-tip='MI Improvement over Random: 28.0% Selection time per 1k features: 46.6 s Datasets: 71 Next to RFImportance: 28.3%, 38.3 s' aria-label='MI Improvement over Random: 28.0% Selection time per 1k features: 46.6 s Datasets: 71 Next to RFImportance: 28.3%, 38.3 s'/><circle class='sa-hit' cx='476.4' cy='55.5' r='10.5' data-tip='CART Improvement over Random: 27.0% Selection time per 1k features: 14.5 s Datasets: 71 Next to mRMR: 26.3%, 16.2 s Next to MarkovBlanket: 25.4%, 19 s' aria-label='CART Improvement over Random: 27.0% Selection time per 1k features: 14.5 s Datasets: 71 Next to mRMR: 26.3%, 16.2 s Next to MarkovBlanket: 25.4%, 19 s'/><circle class='sa-hit' cx='481.0' cy='62.6' r='10.5' data-tip='mRMR Improvement over Random: 26.3% Selection time per 1k features: 16.2 s Datasets: 71 Next to CART: 27.0%, 14.5 s Next to MarkovBlanket: 25.4%, 19 s' aria-label='mRMR Improvement over Random: 26.3% Selection time per 1k features: 16.2 s Datasets: 71 Next to CART: 27.0%, 14.5 s Next to MarkovBlanket: 25.4%, 19 s'/><circle class='sa-hit' cx='691.8' cy='64.5' r='10.5' data-tip='ElasticNet Improvement over Random: 26.2% Selection time per 1k features: 2.14e+03 s Datasets: 71 Next to Lasso: 24.5%, 1.97e+03 s' aria-label='ElasticNet Improvement over Random: 26.2% Selection time per 1k features: 2.14e+03 s Datasets: 71 Next to Lasso: 24.5%, 1.97e+03 s'/><circle class='sa-hit' cx='487.9' cy='72.6' r='10.5' data-tip='MarkovBlanket Improvement over Random: 25.4% Selection time per 1k features: 19 s Datasets: 71 Next to CART: 27.0%, 14.5 s Next to mRMR: 26.3%, 16.2 s' aria-label='MarkovBlanket Improvement over Random: 25.4% Selection time per 1k features: 19 s Datasets: 71 Next to CART: 27.0%, 14.5 s Next to mRMR: 26.3%, 16.2 s'/><circle class='sa-hit' cx='688.2' cy='82.6' r='10.5' data-tip='Lasso Improvement over Random: 24.5% Selection time per 1k features: 1.97e+03 s Datasets: 71 Next to ElasticNet: 26.2%, 2.14e+03 s' aria-label='Lasso Improvement over Random: 24.5% Selection time per 1k features: 1.97e+03 s Datasets: 71 Next to ElasticNet: 26.2%, 2.14e+03 s'/><circle class='sa-hit' cx='782.3' cy='96.1' r='10.5' data-tip='SFS Improvement over Random: 23.3% Selection time per 1k features: 1.75e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 23.3% Selection time per 1k features: 1.75e+04 s Datasets: 71'/><circle class='sa-hit' cx='471.1' cy='104.4' r='10.5' data-tip='GainRatio Improvement over Random: 22.5% Selection time per 1k features: 12.8 s Datasets: 71' aria-label='GainRatio Improvement over Random: 22.5% Selection time per 1k features: 12.8 s Datasets: 71'/><circle class='sa-hit' cx='795.8' cy='154.0' r='10.5' data-tip='(R)ReliefF Improvement over Random: 18.0% Selection time per 1k features: 2.39e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 18.0% Selection time per 1k features: 2.39e+04 s Datasets: 71'/><circle class='sa-hit' cx='738.2' cy='184.3' r='10.5' data-tip='LOCO Improvement over Random: 15.3% Selection time per 1k features: 6.29e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 15.3% Selection time per 1k features: 6.29e+03 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='255.6' r='10.5' data-tip='SBE Improvement over Random: 8.8% Selection time per 1k features: 3.5e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 8.8% Selection time per 1k features: 3.5e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.5' cy='352.2' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0025 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0025 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
F-Test,28.7686,0.6293,71,True
|
| 37 |
RFImportance,28.295,38.3008,71,False
|
| 38 |
MI,27.9958,46.5786,71,False
|
data/subsets/tasks_all/models_linearmodel/budgets_0/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1352.0,1303.0,1413.0,6.54,60.4,17.7,,0.628,0.805,0.2,0.0,71,213
|
| 3 |
-
2,F-Test,Non-search,Correlation,1350.0,1306.0,1398.0,6.6,60.0,19.5,,0.725,0.952,0.0,0.0,71,213
|
| 4 |
-
3,CART,Non-search,Tree,1344.0,1288.0,1406.0,6.66,59.6,19.8,,0.484,0.731,0.0,0.0,71,213
|
| 5 |
-
4,ElasticNet,Non-search,Regularization,1327.0,1279.0,1381.0,6.35,61.8,20.5,,0.509,0.915,6.0,0.0,71,213
|
| 6 |
-
5,MI,Non-search,Information theory,1324.0,1281.0,1368.0,6.75,58.9,20.7,,0.645,0.898,0.4,0.0,71,213
|
| 7 |
-
6,MarkovBlanket,Search,Backward search,1323.0,1282.0,1367.0,6.9,57.8,20.1,,0.418,0.465,0.1,0.0,71,213
|
| 8 |
-
7,mRMR,Non-search,Information theory,1301.0,1253.0,1346.0,7.44,54.0,20.4,,0.528,0.779,0.3,0.0,71,213
|
| 9 |
-
8,SFS,Search,Forward search,1295.0,1243.0,1346.0,7.39,54.4,22.7,,0.368,0.75,970.7,29.1,71,213
|
| 10 |
-
9,Lasso,Non-search,Regularization,1291.0,1244.0,1340.0,7.02,57.0,22.1,,0.421,0.89,1.9,0.0,71,213
|
| 11 |
-
10,GainRatio,Non-search,Information theory,1274.0,1233.0,1318.0,7.81,51.3,24.0,,0.679,0.942,0.2,0.0,71,213
|
| 12 |
-
11,(R)ReliefF,Non-search,Distance,1226.0,1169.0,1282.0,8.6,45.7,25.1,,0.642,0.873,13.7,0.0,71,213
|
| 13 |
-
12,LOCO,Search,Leave-one-out,1197.0,1152.0,1242.0,8.73,44.8,27.7,,0.299,0.665,245.6,0.0,71,213
|
| 14 |
-
13,SBE,Search,Backward search,1168.0,1111.0,1219.0,9.36,40.3,30.0,,0.195,0.621,3600.1,49.3,71,213
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,938.0,1054.0,11.53,24.8,40.2,,-0.001,0.493,0.0,0.0,71,213
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,973.0,890.0,1041.0,12.31,19.2,42.8,,0.588,0.847,0.2,0.0,71,213
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1352.0,1303.0,1413.0,6.54,60.4,17.7,28.3,,0.628,0.805,0.2,0.0,71,213
|
| 3 |
+
2,F-Test,Non-search,Correlation,1350.0,1306.0,1398.0,6.6,60.0,19.5,28.8,,0.725,0.952,0.0,0.0,71,213
|
| 4 |
+
3,CART,Non-search,Tree,1344.0,1288.0,1406.0,6.66,59.6,19.8,27.0,,0.484,0.731,0.0,0.0,71,213
|
| 5 |
+
4,ElasticNet,Non-search,Regularization,1327.0,1279.0,1381.0,6.35,61.8,20.5,26.2,,0.509,0.915,6.0,0.0,71,213
|
| 6 |
+
5,MI,Non-search,Information theory,1324.0,1281.0,1368.0,6.75,58.9,20.7,28.0,,0.645,0.898,0.4,0.0,71,213
|
| 7 |
+
6,MarkovBlanket,Search,Backward search,1323.0,1282.0,1367.0,6.9,57.8,20.1,25.4,,0.418,0.465,0.1,0.0,71,213
|
| 8 |
+
7,mRMR,Non-search,Information theory,1301.0,1253.0,1346.0,7.44,54.0,20.4,26.3,,0.528,0.779,0.3,0.0,71,213
|
| 9 |
+
8,SFS,Search,Forward search,1295.0,1243.0,1346.0,7.39,54.4,22.7,23.3,,0.368,0.75,970.7,29.1,71,213
|
| 10 |
+
9,Lasso,Non-search,Regularization,1291.0,1244.0,1340.0,7.02,57.0,22.1,24.5,,0.421,0.89,1.9,0.0,71,213
|
| 11 |
+
10,GainRatio,Non-search,Information theory,1274.0,1233.0,1318.0,7.81,51.3,24.0,22.5,,0.679,0.942,0.2,0.0,71,213
|
| 12 |
+
11,(R)ReliefF,Non-search,Distance,1226.0,1169.0,1282.0,8.6,45.7,25.1,18.0,,0.642,0.873,13.7,0.0,71,213
|
| 13 |
+
12,LOCO,Search,Leave-one-out,1197.0,1152.0,1242.0,8.73,44.8,27.7,15.3,,0.299,0.665,245.6,0.0,71,213
|
| 14 |
+
13,SBE,Search,Backward search,1168.0,1111.0,1219.0,9.36,40.3,30.0,8.8,,0.195,0.621,3600.1,49.3,71,213
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,938.0,1054.0,11.53,24.8,40.2,0.0,,-0.001,0.493,0.0,0.0,71,213
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,973.0,890.0,1041.0,12.31,19.2,42.8,-12.4,,0.588,0.847,0.2,0.0,71,213
|
data/subsets/tasks_all/models_linearmodel/budgets_1/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1405.0'>1405<span class='sub'>+59 / −56</span></td><td class='num' data-sort='5.71'>5.71</td><td class='num' data-sort='15.8'>15.8</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1377.0'>1377<span class='sub'>+57 / −56</span></td><td class='num' data-sort='5.96'>5.96</td><td class='num' data-sort='15.2'>15.2</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1365.0'>1365<span class='sub'>+61 / −51</span></td><td class='num' data-sort='6.6'>6.60</td><td class='num' data-sort='16.0'>16.0</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1342.0'>1342<span class='sub'>+52 / −45</span></td><td class='num' data-sort='6.59'>6.59</td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1326.0'>1326<span class='sub'>+43 / −46</span></td><td class='num' data-sort='6.86'>6.86</td><td class='num' data-sort='19.4'>19.4</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1318.0'>1318<span class='sub'>+53 / −49</span></td><td class='num' data-sort='7.01'>7.01</td><td class='num' data-sort='20.0'>20.0</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1301.0'>1301<span class='sub'>+46 / −39</span></td><td class='num' data-sort='7.49'>7.49</td><td class='num' data-sort='21.9'>21.9</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1296.0'>1296<span class='sub'>+45 / −44</span></td><td class='num' data-sort='7.46'>7.46</td><td class='num' data-sort='20.6'>20.6</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1280.0'>1280<span class='sub'>+45 / −42</span></td><td class='num' data-sort='7.86'>7.86</td><td class='num' data-sort='23.5'>23.5</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1267.0'>1267<span class='sub'>+51 / −49</span></td><td class='num' data-sort='7.86'>7.86</td><td class='num' data-sort='22.4'>22.4</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1791.6'>1,791.6</td><td class='num' data-sort='43.7'>43.7</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1216.0'>1216<span class='sub'>+48 / −46</span></td><td class='num' data-sort='9.07'>9.07</td><td class='num' data-sort='23.2'>23.2</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='13.0'>13.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1199.0'>1199<span class='sub'>+46 / −46</span></td><td class='num' data-sort='8.97'>8.97</td><td class='num' data-sort='27.4'>27.4</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='235.0'>235.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1184.0'>1184<span class='sub'>+50 / −53</span></td><td class='num' data-sort='9.08'>9.08</td><td class='num' data-sort='29.2'>29.2</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3391.5'>3,391.5</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1026.0'>1026<span class='sub'>+56 / −73</span></td><td class='num' data-sort='11.83'>11.83</td><td class='num' data-sort='40.2'>40.2</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+53 / −60</span></td><td class='num' data-sort='11.66'>11.66</td><td class='num' data-sort='37.5'>37.5</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,ElasticNet,Non-search,Regularization,1405.0,1349.0,1464.0,
|
| 37 |
-
2,Lasso,Non-search,Regularization,1377.0,1321.0,1434.0,
|
| 38 |
-
3,RFImportance,Non-search,Tree,1365.0,1314.0,1426.0,
|
| 39 |
-
4,MI,Non-search,Information theory,1342.0,1297.0,1394.0,
|
| 40 |
-
5,F-Test,Non-search,Correlation,1326.0,1280.0,1369.0,
|
| 41 |
-
6,CART,Non-search,Tree,1318.0,1269.0,1371.0,
|
| 42 |
-
7,MarkovBlanket,Search,Backward search,1301.0,1262.0,1347.0,
|
| 43 |
-
8,mRMR,Non-search,Information theory,1296.0,1252.0,1341.0,
|
| 44 |
-
9,GainRatio,Non-search,Information theory,1280.0,1238.0,1325.0,
|
| 45 |
-
10,SFS,Search,Forward search,1267.0,1218.0,1318.0,
|
| 46 |
-
11,(R)ReliefF,Non-search,Distance,1216.0,1170.0,1264.0,
|
| 47 |
-
12,LOCO,Search,Leave-one-out,1199.0,1153.0,1245.0,
|
| 48 |
-
13,SBE,Search,Backward search,1184.0,1131.0,1234.0,
|
| 49 |
-
14,LaplacianScore,Non-search,Distance,1026.0,953.0,1082.0,
|
| 50 |
-
15,Random,Baseline,Baseline,1000.0,940.0,1053.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1405.0'>1405<span class='sub'>+59 / −56</span></td><td class='num' data-sort='25.9'>25.9</td><td class='num shaded' data-sort='0.519' style='background:rgba(0,114,178,0.43)'>0.519</td><td class='num shaded' data-sort='0.873' style='background:rgba(213,94,0,0.66)'>0.873</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1377.0'>1377<span class='sub'>+57 / −56</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='2.0'>2.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1365.0'>1365<span class='sub'>+61 / −51</span></td><td class='num' data-sort='27.1'>27.1</td><td class='num shaded' data-sort='0.653' style='background:rgba(0,114,178,0.52)'>0.653</td><td class='num shaded' data-sort='0.741' style='background:rgba(213,94,0,0.58)'>0.741</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1342.0'>1342<span class='sub'>+52 / −45</span></td><td class='num' data-sort='26.2'>26.2</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.863' style='background:rgba(213,94,0,0.66)'>0.863</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1326.0'>1326<span class='sub'>+43 / −46</span></td><td class='num' data-sort='25.5'>25.5</td><td class='num shaded' data-sort='0.734' style='background:rgba(0,114,178,0.57)'>0.734</td><td class='num shaded' data-sort='0.932' style='background:rgba(213,94,0,0.70)'>0.932</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1318.0'>1318<span class='sub'>+53 / −49</span></td><td class='num' data-sort='24.1'>24.1</td><td class='num shaded' data-sort='0.456' style='background:rgba(0,114,178,0.39)'>0.456</td><td class='num shaded' data-sort='0.638' style='background:rgba(213,94,0,0.51)'>0.638</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1301.0'>1301<span class='sub'>+46 / −39</span></td><td class='num' data-sort='21.4'>21.4</td><td class='num shaded' data-sort='0.396' style='background:rgba(0,114,178,0.35)'>0.396</td><td class='num shaded' data-sort='0.467' style='background:rgba(213,94,0,0.39)'>0.467</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1296.0'>1296<span class='sub'>+45 / −44</span></td><td class='num' data-sort='24.4'>24.4</td><td class='num shaded' data-sort='0.578' style='background:rgba(0,114,178,0.47)'>0.578</td><td class='num shaded' data-sort='0.712' style='background:rgba(213,94,0,0.56)'>0.712</td><td class='num' data-sort='0.3'>0.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1280.0'>1280<span class='sub'>+45 / −42</span></td><td class='num' data-sort='21.6'>21.6</td><td class='num shaded' data-sort='0.681' style='background:rgba(0,114,178,0.54)'>0.681</td><td class='num shaded' data-sort='0.915' style='background:rgba(213,94,0,0.69)'>0.915</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1267.0'>1267<span class='sub'>+51 / −49</span></td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.278' style='background:rgba(0,114,178,0.27)'>0.278</td><td class='num shaded' data-sort='0.664' style='background:rgba(213,94,0,0.52)'>0.664</td><td class='num' data-sort='1791.6'>1,791.6</td><td class='num' data-sort='43.7'>43.7</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1216.0'>1216<span class='sub'>+48 / −46</span></td><td class='num' data-sort='16.8'>16.8</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.808' style='background:rgba(213,94,0,0.62)'>0.808</td><td class='num' data-sort='13.0'>13.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1199.0'>1199<span class='sub'>+46 / −46</span></td><td class='num' data-sort='12.5'>12.5</td><td class='num shaded' data-sort='0.22' style='background:rgba(0,114,178,0.23)'>0.220</td><td class='num shaded' data-sort='0.613' style='background:rgba(213,94,0,0.49)'>0.613</td><td class='num' data-sort='235.0'>235.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1184.0'>1184<span class='sub'>+50 / −53</span></td><td class='num' data-sort='10.8'>10.8</td><td class='num shaded' data-sort='0.175' style='background:rgba(0,114,178,0.20)'>0.175</td><td class='num shaded' data-sort='0.578' style='background:rgba(213,94,0,0.47)'>0.578</td><td class='num' data-sort='3391.5'>3,391.5</td><td class='num' data-sort='49.3'>49.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1026.0'>1026<span class='sub'>+56 / −73</span></td><td class='num' data-sort='-22.4'>-22.4</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.85' style='background:rgba(213,94,0,0.65)'>0.850</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+53 / −60</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.499' style='background:rgba(213,94,0,0.41)'>0.499</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,ElasticNet,Non-search,Regularization,1405.0,1349.0,1464.0,25.9,0.519,0.873,6.3,0.0
|
| 37 |
+
2,Lasso,Non-search,Regularization,1377.0,1321.0,1434.0,25.5,0.422,0.825,2.0,0.0
|
| 38 |
+
3,RFImportance,Non-search,Tree,1365.0,1314.0,1426.0,27.1,0.653,0.741,0.2,0.0
|
| 39 |
+
4,MI,Non-search,Information theory,1342.0,1297.0,1394.0,26.2,0.673,0.863,0.4,0.0
|
| 40 |
+
5,F-Test,Non-search,Correlation,1326.0,1280.0,1369.0,25.5,0.734,0.932,0.0,0.0
|
| 41 |
+
6,CART,Non-search,Tree,1318.0,1269.0,1371.0,24.1,0.456,0.638,0.0,0.0
|
| 42 |
+
7,MarkovBlanket,Search,Backward search,1301.0,1262.0,1347.0,21.4,0.396,0.467,0.1,0.0
|
| 43 |
+
8,mRMR,Non-search,Information theory,1296.0,1252.0,1341.0,24.4,0.578,0.712,0.3,0.0
|
| 44 |
+
9,GainRatio,Non-search,Information theory,1280.0,1238.0,1325.0,21.6,0.681,0.915,0.2,0.0
|
| 45 |
+
10,SFS,Search,Forward search,1267.0,1218.0,1318.0,19.8,0.278,0.664,1791.6,43.7
|
| 46 |
+
11,(R)ReliefF,Non-search,Distance,1216.0,1170.0,1264.0,16.8,0.647,0.808,13.0,0.0
|
| 47 |
+
12,LOCO,Search,Leave-one-out,1199.0,1153.0,1245.0,12.5,0.22,0.613,235.0,0.0
|
| 48 |
+
13,SBE,Search,Backward search,1184.0,1131.0,1234.0,10.8,0.175,0.578,3391.5,49.3
|
| 49 |
+
14,LaplacianScore,Non-search,Distance,1026.0,953.0,1082.0,-22.4,0.644,0.85,0.2,0.0
|
| 50 |
+
15,Random,Baseline,Baseline,1000.0,940.0,1053.0,0.0,0.001,0.499,0.0,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_linearmodel/budgets_1/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='159.3' x2='159.3' y1='16.2' y2='368.0'/><text class='sa-text' x='159.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='259.4' x2='259.4' y1='16.2' y2='368.0'/><text class='sa-text' x='259.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='359.5' x2='359.5' y1='16.2' y2='368.0'/><text class='sa-text' x='359.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='459.6' x2='459.6' y1='16.2' y2='368.0'/><text class='sa-text' x='459.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.7' x2='559.7' y1='16.2' y2='368.0'/><text class='sa-text' x='559.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='659.7' x2='659.7' y1='16.2' y2='368.0'/><text class='sa-text' x='659.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='759.8' x2='759.8' y1='16.2' y2='368.0'/><text class='sa-text' x='759.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='292.9' y2='292.9'/><text class='sa-text' x='57.9' y='298.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='233.8' y2='233.8'/><text class='sa-text' x='57.9' y='239.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='174.7' y2='174.7'/><text class='sa-text' x='57.9' y='180.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='115.6' y2='115.6'/><text class='sa-text' x='57.9' y='121.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='56.5' y2='56.5'/><text class='sa-text' x='57.9' y='62.5' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 340.6,50.3 518.4,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='518.4' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='526.0' cy='42.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.4' cy='46.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='340.6' cy='50.3' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='690.3' cy='50.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='492.1' cy='63.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='475.9' cy='66.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='469.8' cy='97.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='488.6' cy='99.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='799.7' cy='117.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.5' cy='154.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='740.2' cy='204.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='223.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='518.4' y1='32.2' x2='407.6' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='407.6' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='526.0' y1='42.3' x2='405.7' y2='54.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='405.7' y='54.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='691.4' y1='46.4' x2='738.2' y2='32.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='738.2' y='32.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='340.6' y1='50.3' x2='302.2' y2='64.0' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='302.2' y='64.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='690.3' y1='50.5' x2='658.0' y2='64.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='658.0' y='64.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='492.1' y1='63.3' x2='550.1' y2='57.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='550.1' y='57.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='475.9' y1='66.6' x2='430.6' y2='88.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='430.6' y='88.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='469.8' y1='97.3' x2='428.3' y2='116.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='428.3' y='116.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='488.6' y1='99.6' x2='595.7' y2='150.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='595.7' y='150.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='799.7' y1='117.6' x2='820.9' y2='103.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='820.9' y='103.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><text class='sa-dim sa-halo' x='796.9' y='139.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='740.2' y1='204.1' x2='707.5' y2='217.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='707.5' y='217.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='238.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='518.4' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 27.1% Selection time per 1k features: 38.7 s Datasets: 71 Next to MI: 26.2%, 46.1 s' aria-label='RFImportance (on the Pareto front) Improvement over Random: 27.1% Selection time per 1k features: 38.7 s Datasets: 71 Next to MI: 26.2%, 46.1 s'/><circle class='sa-hit' cx='526.0' cy='42.3' r='10.5' data-tip='MI Improvement over Random: 26.2% Selection time per 1k features: 46.1 s Datasets: 71 Next to RFImportance: 27.1%, 38.7 s' aria-label='MI Improvement over Random: 26.2% Selection time per 1k features: 46.1 s Datasets: 71 Next to RFImportance: 27.1%, 38.7 s'/><circle class='sa-hit' cx='691.4' cy='46.4' r='10.5' data-tip='ElasticNet Improvement over Random: 25.9% Selection time per 1k features: 2.07e+03 s Datasets: 71 Next to Lasso: 25.5%, 2.02e+03 s' aria-label='ElasticNet Improvement over Random: 25.9% Selection time per 1k features: 2.07e+03 s Datasets: 71 Next to Lasso: 25.5%, 2.02e+03 s'/><circle class='sa-hit' cx='340.6' cy='50.3' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 25.5% Selection time per 1k features: 0.648 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 25.5% Selection time per 1k features: 0.648 s Datasets: 71'/><circle class='sa-hit' cx='690.3' cy='50.5' r='10.5' data-tip='Lasso Improvement over Random: 25.5% Selection time per 1k features: 2.02e+03 s Datasets: 71 Next to ElasticNet: 25.9%, 2.07e+03 s' aria-label='Lasso Improvement over Random: 25.5% Selection time per 1k features: 2.02e+03 s Datasets: 71 Next to ElasticNet: 25.9%, 2.07e+03 s'/><circle class='sa-hit' cx='492.1' cy='63.3' r='10.5' data-tip='mRMR Improvement over Random: 24.4% Selection time per 1k features: 21.2 s Datasets: 71 Next to CART: 24.1%, 14.6 s' aria-label='mRMR Improvement over Random: 24.4% Selection time per 1k features: 21.2 s Datasets: 71 Next to CART: 24.1%, 14.6 s'/><circle class='sa-hit' cx='475.9' cy='66.6' r='10.5' data-tip='CART Improvement over Random: 24.1% Selection time per 1k features: 14.6 s Datasets: 71 Next to mRMR: 24.4%, 21.2 s' aria-label='CART Improvement over Random: 24.1% Selection time per 1k features: 14.6 s Datasets: 71 Next to mRMR: 24.4%, 21.2 s'/><circle class='sa-hit' cx='469.8' cy='97.3' r='10.5' data-tip='GainRatio Improvement over Random: 21.6% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 21.4%, 19.5 s' aria-label='GainRatio Improvement over Random: 21.6% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 21.4%, 19.5 s'/><circle class='sa-hit' cx='488.6' cy='99.6' r='10.5' data-tip='MarkovBlanket Improvement over Random: 21.4% Selection time per 1k features: 19.5 s Datasets: 71 Next to GainRatio: 21.6%, 12.7 s' aria-label='MarkovBlanket Improvement over Random: 21.4% Selection time per 1k features: 19.5 s Datasets: 71 Next to GainRatio: 21.6%, 12.7 s'/><circle class='sa-hit' cx='799.7' cy='117.6' r='10.5' data-tip='SFS Improvement over Random: 19.8% Selection time per 1k features: 2.5e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 19.8% Selection time per 1k features: 2.5e+04 s Datasets: 71'/><circle class='sa-hit' cx='797.5' cy='154.0' r='10.5' data-tip='(R)ReliefF Improvement over Random: 16.8% Selection time per 1k features: 2.38e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 16.8% Selection time per 1k features: 2.38e+04 s Datasets: 71'/><circle class='sa-hit' cx='740.2' cy='204.1' r='10.5' data-tip='LOCO Improvement over Random: 12.5% Selection time per 1k features: 6.37e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 12.5% Selection time per 1k features: 6.37e+03 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='223.9' r='10.5' data-tip='SBE Improvement over Random: 10.8% Selection time per 1k features: 3.35e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 10.8% Selection time per 1k features: 3.35e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,27.0594,38.7475,71,True
|
| 37 |
MI,26.2027,46.0868,71,False
|
| 38 |
ElasticNet,25.8586,2071.727,71,False
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='159.3' x2='159.3' y1='16.2' y2='368.0'/><text class='sa-text' x='159.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='259.4' x2='259.4' y1='16.2' y2='368.0'/><text class='sa-text' x='259.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='359.5' x2='359.5' y1='16.2' y2='368.0'/><text class='sa-text' x='359.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='459.6' x2='459.6' y1='16.2' y2='368.0'/><text class='sa-text' x='459.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.7' x2='559.7' y1='16.2' y2='368.0'/><text class='sa-text' x='559.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='659.7' x2='659.7' y1='16.2' y2='368.0'/><text class='sa-text' x='659.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='759.8' x2='759.8' y1='16.2' y2='368.0'/><text class='sa-text' x='759.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='292.9' y2='292.9'/><text class='sa-text' x='57.9' y='298.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='233.8' y2='233.8'/><text class='sa-text' x='57.9' y='239.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='174.7' y2='174.7'/><text class='sa-text' x='57.9' y='180.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='115.6' y2='115.6'/><text class='sa-text' x='57.9' y='121.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='56.5' y2='56.5'/><text class='sa-text' x='57.9' y='62.5' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 340.6,50.3 518.4,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='518.4' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='526.0' cy='42.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='691.4' cy='46.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='340.6' cy='50.3' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='690.3' cy='50.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='492.1' cy='63.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='475.9' cy='66.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='469.8' cy='97.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='488.6' cy='99.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='799.7' cy='117.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.5' cy='154.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='740.2' cy='204.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='223.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='518.4' y1='32.2' x2='402.6' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='402.6' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='526.0' y1='42.3' x2='399.4' y2='59.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='399.4' y='59.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='691.4' y1='46.4' x2='738.2' y2='32.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='738.2' y='32.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='340.6' y1='50.3' x2='301.0' y2='64.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='301.0' y='64.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='690.3' y1='50.5' x2='656.8' y2='65.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='656.8' y='65.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='492.1' y1='63.3' x2='550.1' y2='56.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='550.1' y='56.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='475.9' y1='66.6' x2='426.8' y2='85.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='426.8' y='85.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='469.8' y1='97.3' x2='418.2' y2='121.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='418.2' y='121.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='488.6' y1='99.6' x2='601.3' y2='159.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='601.3' y='159.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='799.7' y1='117.6' x2='820.9' y2='103.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='820.9' y='103.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><text class='sa-dim sa-halo' x='796.9' y='135.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='740.2' y1='204.1' x2='706.2' y2='218.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='706.2' y='218.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='237.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='518.4' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 27.1% Selection time per 1k features: 38.7 s Datasets: 71 Next to MI: 26.2%, 46.1 s' aria-label='RFImportance (on the Pareto front) Improvement over Random: 27.1% Selection time per 1k features: 38.7 s Datasets: 71 Next to MI: 26.2%, 46.1 s'/><circle class='sa-hit' cx='526.0' cy='42.3' r='10.5' data-tip='MI Improvement over Random: 26.2% Selection time per 1k features: 46.1 s Datasets: 71 Next to RFImportance: 27.1%, 38.7 s' aria-label='MI Improvement over Random: 26.2% Selection time per 1k features: 46.1 s Datasets: 71 Next to RFImportance: 27.1%, 38.7 s'/><circle class='sa-hit' cx='691.4' cy='46.4' r='10.5' data-tip='ElasticNet Improvement over Random: 25.9% Selection time per 1k features: 2.07e+03 s Datasets: 71 Next to Lasso: 25.5%, 2.02e+03 s' aria-label='ElasticNet Improvement over Random: 25.9% Selection time per 1k features: 2.07e+03 s Datasets: 71 Next to Lasso: 25.5%, 2.02e+03 s'/><circle class='sa-hit' cx='340.6' cy='50.3' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 25.5% Selection time per 1k features: 0.648 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 25.5% Selection time per 1k features: 0.648 s Datasets: 71'/><circle class='sa-hit' cx='690.3' cy='50.5' r='10.5' data-tip='Lasso Improvement over Random: 25.5% Selection time per 1k features: 2.02e+03 s Datasets: 71 Next to ElasticNet: 25.9%, 2.07e+03 s' aria-label='Lasso Improvement over Random: 25.5% Selection time per 1k features: 2.02e+03 s Datasets: 71 Next to ElasticNet: 25.9%, 2.07e+03 s'/><circle class='sa-hit' cx='492.1' cy='63.3' r='10.5' data-tip='mRMR Improvement over Random: 24.4% Selection time per 1k features: 21.2 s Datasets: 71 Next to CART: 24.1%, 14.6 s' aria-label='mRMR Improvement over Random: 24.4% Selection time per 1k features: 21.2 s Datasets: 71 Next to CART: 24.1%, 14.6 s'/><circle class='sa-hit' cx='475.9' cy='66.6' r='10.5' data-tip='CART Improvement over Random: 24.1% Selection time per 1k features: 14.6 s Datasets: 71 Next to mRMR: 24.4%, 21.2 s' aria-label='CART Improvement over Random: 24.1% Selection time per 1k features: 14.6 s Datasets: 71 Next to mRMR: 24.4%, 21.2 s'/><circle class='sa-hit' cx='469.8' cy='97.3' r='10.5' data-tip='GainRatio Improvement over Random: 21.6% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 21.4%, 19.5 s' aria-label='GainRatio Improvement over Random: 21.6% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 21.4%, 19.5 s'/><circle class='sa-hit' cx='488.6' cy='99.6' r='10.5' data-tip='MarkovBlanket Improvement over Random: 21.4% Selection time per 1k features: 19.5 s Datasets: 71 Next to GainRatio: 21.6%, 12.7 s' aria-label='MarkovBlanket Improvement over Random: 21.4% Selection time per 1k features: 19.5 s Datasets: 71 Next to GainRatio: 21.6%, 12.7 s'/><circle class='sa-hit' cx='799.7' cy='117.6' r='10.5' data-tip='SFS Improvement over Random: 19.8% Selection time per 1k features: 2.5e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 19.8% Selection time per 1k features: 2.5e+04 s Datasets: 71'/><circle class='sa-hit' cx='797.5' cy='154.0' r='10.5' data-tip='(R)ReliefF Improvement over Random: 16.8% Selection time per 1k features: 2.38e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 16.8% Selection time per 1k features: 2.38e+04 s Datasets: 71'/><circle class='sa-hit' cx='740.2' cy='204.1' r='10.5' data-tip='LOCO Improvement over Random: 12.5% Selection time per 1k features: 6.37e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 12.5% Selection time per 1k features: 6.37e+03 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='223.9' r='10.5' data-tip='SBE Improvement over Random: 10.8% Selection time per 1k features: 3.35e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 10.8% Selection time per 1k features: 3.35e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,27.0594,38.7475,71,True
|
| 37 |
MI,26.2027,46.0868,71,False
|
| 38 |
ElasticNet,25.8586,2071.727,71,False
|
data/subsets/tasks_all/models_linearmodel/budgets_1/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,ElasticNet,Non-search,Regularization,1405.0,1349.0,1464.0,5.71,66.4,15.8,,0.519,0.873,6.3,0.0,71,213
|
| 3 |
-
2,Lasso,Non-search,Regularization,1377.0,1321.0,1434.0,5.96,64.6,15.2,,0.422,0.825,2.0,0.0,71,213
|
| 4 |
-
3,RFImportance,Non-search,Tree,1365.0,1314.0,1426.0,6.6,60.0,16.0,,0.653,0.741,0.2,0.0,71,213
|
| 5 |
-
4,MI,Non-search,Information theory,1342.0,1297.0,1394.0,6.59,60.1,17.7,,0.673,0.863,0.4,0.0,71,213
|
| 6 |
-
5,F-Test,Non-search,Correlation,1326.0,1280.0,1369.0,6.86,58.1,19.4,,0.734,0.932,0.0,0.0,71,213
|
| 7 |
-
6,CART,Non-search,Tree,1318.0,1269.0,1371.0,7.01,57.1,20.0,,0.456,0.638,0.0,0.0,71,213
|
| 8 |
-
7,MarkovBlanket,Search,Backward search,1301.0,1262.0,1347.0,7.49,53.7,21.9,,0.396,0.467,0.1,0.0,71,213
|
| 9 |
-
8,mRMR,Non-search,Information theory,1296.0,1252.0,1341.0,7.46,53.9,20.6,,0.578,0.712,0.3,0.0,71,213
|
| 10 |
-
9,GainRatio,Non-search,Information theory,1280.0,1238.0,1325.0,7.86,51.0,23.5,,0.681,0.915,0.2,0.0,71,213
|
| 11 |
-
10,SFS,Search,Forward search,1267.0,1218.0,1318.0,7.86,51.0,22.4,,0.278,0.664,1791.6,43.7,71,213
|
| 12 |
-
11,(R)ReliefF,Non-search,Distance,1216.0,1170.0,1264.0,9.07,42.4,23.2,,0.647,0.808,13.0,0.0,71,213
|
| 13 |
-
12,LOCO,Search,Leave-one-out,1199.0,1153.0,1245.0,8.97,43.1,27.4,,0.22,0.613,235.0,0.0,71,213
|
| 14 |
-
13,SBE,Search,Backward search,1184.0,1131.0,1234.0,9.08,42.2,29.2,,0.175,0.578,3391.5,49.3,71,213
|
| 15 |
-
14,LaplacianScore,Non-search,Distance,1026.0,953.0,1082.0,11.83,22.7,40.2,,0.644,0.85,0.2,0.0,71,213
|
| 16 |
-
15,Random,Baseline,Baseline,1000.0,940.0,1053.0,11.66,23.8,37.5,,0.001,0.499,0.0,0.0,71,213
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,ElasticNet,Non-search,Regularization,1405.0,1349.0,1464.0,5.71,66.4,15.8,25.9,,0.519,0.873,6.3,0.0,71,213
|
| 3 |
+
2,Lasso,Non-search,Regularization,1377.0,1321.0,1434.0,5.96,64.6,15.2,25.5,,0.422,0.825,2.0,0.0,71,213
|
| 4 |
+
3,RFImportance,Non-search,Tree,1365.0,1314.0,1426.0,6.6,60.0,16.0,27.1,,0.653,0.741,0.2,0.0,71,213
|
| 5 |
+
4,MI,Non-search,Information theory,1342.0,1297.0,1394.0,6.59,60.1,17.7,26.2,,0.673,0.863,0.4,0.0,71,213
|
| 6 |
+
5,F-Test,Non-search,Correlation,1326.0,1280.0,1369.0,6.86,58.1,19.4,25.5,,0.734,0.932,0.0,0.0,71,213
|
| 7 |
+
6,CART,Non-search,Tree,1318.0,1269.0,1371.0,7.01,57.1,20.0,24.1,,0.456,0.638,0.0,0.0,71,213
|
| 8 |
+
7,MarkovBlanket,Search,Backward search,1301.0,1262.0,1347.0,7.49,53.7,21.9,21.4,,0.396,0.467,0.1,0.0,71,213
|
| 9 |
+
8,mRMR,Non-search,Information theory,1296.0,1252.0,1341.0,7.46,53.9,20.6,24.4,,0.578,0.712,0.3,0.0,71,213
|
| 10 |
+
9,GainRatio,Non-search,Information theory,1280.0,1238.0,1325.0,7.86,51.0,23.5,21.6,,0.681,0.915,0.2,0.0,71,213
|
| 11 |
+
10,SFS,Search,Forward search,1267.0,1218.0,1318.0,7.86,51.0,22.4,19.8,,0.278,0.664,1791.6,43.7,71,213
|
| 12 |
+
11,(R)ReliefF,Non-search,Distance,1216.0,1170.0,1264.0,9.07,42.4,23.2,16.8,,0.647,0.808,13.0,0.0,71,213
|
| 13 |
+
12,LOCO,Search,Leave-one-out,1199.0,1153.0,1245.0,8.97,43.1,27.4,12.5,,0.22,0.613,235.0,0.0,71,213
|
| 14 |
+
13,SBE,Search,Backward search,1184.0,1131.0,1234.0,9.08,42.2,29.2,10.8,,0.175,0.578,3391.5,49.3,71,213
|
| 15 |
+
14,LaplacianScore,Non-search,Distance,1026.0,953.0,1082.0,11.83,22.7,40.2,-22.4,,0.644,0.85,0.2,0.0,71,213
|
| 16 |
+
15,Random,Baseline,Baseline,1000.0,940.0,1053.0,11.66,23.8,37.5,0.0,,0.001,0.499,0.0,0.0,71,213
|
data/subsets/tasks_all/models_linearmodel/budgets_2/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1313.0'>1313<span class='sub'>+63 / −56</span></td><td class='num' data-sort='5.75'>5.75</td><td class='num' data-sort='14.0'>14.0</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+57 / −55</span></td><td class='num' data-sort='6.11'>6.11</td><td class='num' data-sort='14.2'>14.2</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1263.0'>1263<span class='sub'>+58 / −50</span></td><td class='num' data-sort='6.51'>6.51</td><td class='num' data-sort='14.9'>14.9</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1258.0'>1258<span class='sub'>+45 / −45</span></td><td class='num' data-sort='6.65'>6.65</td><td class='num' data-sort='16.8'>16.8</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1216.0'>1216<span class='sub'>+46 / −45</span></td><td class='num' data-sort='7.2'>7.20</td><td class='num' data-sort='17.6'>17.6</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1203.0'>1203<span class='sub'>+55 / −49</span></td><td class='num' data-sort='7.48'>7.48</td><td class='num' data-sort='19.7'>19.7</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1185.0'>1185<span class='sub'>+45 / −46</span></td><td class='num' data-sort='8.04'>8.04</td><td class='num' data-sort='20.8'>20.8</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1173.0'>1173<span class='sub'>+48 / −50</span></td><td class='num' data-sort='7.81'>7.81</td><td class='num' data-sort='19.1'>19.1</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+42 / −43</span></td><td class='num' data-sort='8.03'>8.03</td><td class='num' data-sort='21.5'>21.5</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+43 / −46</span></td><td class='num' data-sort='7.82'>7.82</td><td class='num' data-sort='20.7'>20.7</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2264.7'>2,264.7</td><td class='num' data-sort='47.9'>47.9</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+44 / −45</span></td><td class='num' data-sort='8.41'>8.41</td><td class='num' data-sort='19.5'>19.5</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='12.8'>12.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1109.0'>1109<span class='sub'>+41 / −48</span></td><td class='num' data-sort='8.81'>8.81</td><td class='num' data-sort='25.7'>25.7</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='6.1'>6.1</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1088.0'>1088<span class='sub'>+47 / −49</span></td><td class='num' data-sort='8.94'>8.94</td><td class='num' data-sort='25.1'>25.1</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2787.0'>2,787.0</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+56 / −65</span></td><td class='num' data-sort='10.69'>10.69</td><td class='num' data-sort='31.5'>31.5</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='942.0'>942<span class='sub'>+55 / −74</span></td><td class='num' data-sort='11.74'>11.74</td><td class='num' data-sort='37.1'>37.1</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,ElasticNet,Non-search,Regularization,1313.0,1257.0,1376.0,
|
| 37 |
-
2,Lasso,Non-search,Regularization,1274.0,1219.0,1331.0,
|
| 38 |
-
3,RFImportance,Non-search,Tree,1263.0,1213.0,1321.0,
|
| 39 |
-
4,MI,Non-search,Information theory,1258.0,1213.0,1303.0,
|
| 40 |
-
5,F-Test,Non-search,Correlation,1216.0,1171.0,1262.0,
|
| 41 |
-
6,CART,Non-search,Tree,1203.0,1154.0,1258.0,
|
| 42 |
-
7,GainRatio,Non-search,Information theory,1185.0,1139.0,1230.0,
|
| 43 |
-
8,mRMR,Non-search,Information theory,1173.0,1123.0,1221.0,
|
| 44 |
-
9,MarkovBlanket,Search,Backward search,1172.0,1129.0,1214.0,
|
| 45 |
-
10,SFS,Search,Forward search,1172.0,1126.0,1215.0,
|
| 46 |
-
11,(R)ReliefF,Non-search,Distance,1168.0,1123.0,1212.0,
|
| 47 |
-
12,LOCO,Search,Leave-one-out,1109.0,1061.0,1150.0,8.
|
| 48 |
-
13,SBE,Search,Backward search,1088.0,1039.0,1135.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,935.0,1056.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,942.0,868.0,997.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1313.0'>1313<span class='sub'>+63 / −56</span></td><td class='num' data-sort='20.6'>20.6</td><td class='num shaded' data-sort='0.481' style='background:rgba(0,114,178,0.40)'>0.481</td><td class='num shaded' data-sort='0.777' style='background:rgba(213,94,0,0.60)'>0.777</td><td class='num' data-sort='6.3'>6.3</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1274.0'>1274<span class='sub'>+57 / −55</span></td><td class='num' data-sort='19.4'>19.4</td><td class='num shaded' data-sort='0.389' style='background:rgba(0,114,178,0.34)'>0.389</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1263.0'>1263<span class='sub'>+58 / −50</span></td><td class='num' data-sort='22.2'>22.2</td><td class='num shaded' data-sort='0.676' style='background:rgba(0,114,178,0.53)'>0.676</td><td class='num shaded' data-sort='0.691' style='background:rgba(213,94,0,0.54)'>0.691</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1258.0'>1258<span class='sub'>+45 / −45</span></td><td class='num' data-sort='20.9'>20.9</td><td class='num shaded' data-sort='0.683' style='background:rgba(0,114,178,0.54)'>0.683</td><td class='num shaded' data-sort='0.815' style='background:rgba(213,94,0,0.63)'>0.815</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1216.0'>1216<span class='sub'>+46 / −45</span></td><td class='num' data-sort='18.6'>18.6</td><td class='num shaded' data-sort='0.706' style='background:rgba(0,114,178,0.55)'>0.706</td><td class='num shaded' data-sort='0.879' style='background:rgba(213,94,0,0.67)'>0.879</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1203.0'>1203<span class='sub'>+55 / −49</span></td><td class='num' data-sort='17.5'>17.5</td><td class='num shaded' data-sort='0.444' style='background:rgba(0,114,178,0.38)'>0.444</td><td class='num shaded' data-sort='0.572' style='background:rgba(213,94,0,0.46)'>0.572</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1185.0'>1185<span class='sub'>+45 / −46</span></td><td class='num' data-sort='15.0'>15.0</td><td class='num shaded' data-sort='0.67' style='background:rgba(0,114,178,0.53)'>0.670</td><td class='num shaded' data-sort='0.856' style='background:rgba(213,94,0,0.65)'>0.856</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1173.0'>1173<span class='sub'>+48 / −50</span></td><td class='num' data-sort='17.9'>17.9</td><td class='num shaded' data-sort='0.614' style='background:rgba(0,114,178,0.49)'>0.614</td><td class='num shaded' data-sort='0.66' style='background:rgba(213,94,0,0.52)'>0.660</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+42 / −43</span></td><td class='num' data-sort='14.8'>14.8</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.447' style='background:rgba(213,94,0,0.38)'>0.447</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1172.0'>1172<span class='sub'>+43 / −46</span></td><td class='num' data-sort='13.8'>13.8</td><td class='num shaded' data-sort='0.239' style='background:rgba(0,114,178,0.24)'>0.239</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2264.7'>2,264.7</td><td class='num' data-sort='47.9'>47.9</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1168.0'>1168<span class='sub'>+44 / −45</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.673' style='background:rgba(0,114,178,0.53)'>0.673</td><td class='num shaded' data-sort='0.757' style='background:rgba(213,94,0,0.59)'>0.757</td><td class='num' data-sort='12.8'>12.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1109.0'>1109<span class='sub'>+41 / −48</span></td><td class='num' data-sort='8.1'>8.1</td><td class='num shaded' data-sort='0.151' style='background:rgba(0,114,178,0.18)'>0.151</td><td class='num shaded' data-sort='0.575' style='background:rgba(213,94,0,0.47)'>0.575</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='6.1'>6.1</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1088.0'>1088<span class='sub'>+47 / −49</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.144' style='background:rgba(0,114,178,0.18)'>0.144</td><td class='num shaded' data-sort='0.556' style='background:rgba(213,94,0,0.45)'>0.556</td><td class='num' data-sort='2787.0'>2,787.0</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+56 / −65</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='-0.001' style='background:rgba(0,114,178,0.08)'>-0.001</td><td class='num shaded' data-sort='0.513' style='background:rgba(213,94,0,0.42)'>0.513</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='942.0'>942<span class='sub'>+55 / −74</span></td><td class='num' data-sort='-52.8'>-52.8</td><td class='num shaded' data-sort='0.66' style='background:rgba(0,114,178,0.52)'>0.660</td><td class='num shaded' data-sort='0.827' style='background:rgba(213,94,0,0.63)'>0.827</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,ElasticNet,Non-search,Regularization,1313.0,1257.0,1376.0,20.6,0.481,0.777,6.3,0.0
|
| 37 |
+
2,Lasso,Non-search,Regularization,1274.0,1219.0,1331.0,19.4,0.389,0.73,1.8,0.0
|
| 38 |
+
3,RFImportance,Non-search,Tree,1263.0,1213.0,1321.0,22.2,0.676,0.691,0.2,0.0
|
| 39 |
+
4,MI,Non-search,Information theory,1258.0,1213.0,1303.0,20.9,0.683,0.815,0.4,0.0
|
| 40 |
+
5,F-Test,Non-search,Correlation,1216.0,1171.0,1262.0,18.6,0.706,0.879,0.0,0.0
|
| 41 |
+
6,CART,Non-search,Tree,1203.0,1154.0,1258.0,17.5,0.444,0.572,0.1,0.0
|
| 42 |
+
7,GainRatio,Non-search,Information theory,1185.0,1139.0,1230.0,15.0,0.67,0.856,0.2,0.0
|
| 43 |
+
8,mRMR,Non-search,Information theory,1173.0,1123.0,1221.0,17.9,0.614,0.66,0.4,0.0
|
| 44 |
+
9,MarkovBlanket,Search,Backward search,1172.0,1129.0,1214.0,14.8,0.413,0.447,0.1,0.0
|
| 45 |
+
10,SFS,Search,Forward search,1172.0,1126.0,1215.0,13.8,0.239,0.627,2264.7,47.9
|
| 46 |
+
11,(R)ReliefF,Non-search,Distance,1168.0,1123.0,1212.0,17.7,0.673,0.757,12.8,0.0
|
| 47 |
+
12,LOCO,Search,Leave-one-out,1109.0,1061.0,1150.0,8.1,0.151,0.575,242.0,6.1
|
| 48 |
+
13,SBE,Search,Backward search,1088.0,1039.0,1135.0,7.5,0.144,0.556,2787.0,48.4
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,935.0,1056.0,0.0,-0.001,0.513,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,942.0,868.0,997.0,-52.8,0.66,0.827,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_linearmodel/budgets_2/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='161.2' x2='161.2' y1='16.2' y2='368.0'/><text class='sa-text' x='161.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='261.7' x2='261.7' y1='16.2' y2='368.0'/><text class='sa-text' x='261.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.2' x2='362.2' y1='16.2' y2='368.0'/><text class='sa-text' x='362.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='462.7' x2='462.7' y1='16.2' y2='368.0'/><text class='sa-text' x='462.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='563.2' x2='563.2' y1='16.2' y2='368.0'/><text class='sa-text' x='563.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.7' x2='663.7' y1='16.2' y2='368.0'/><text class='sa-text' x='663.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='764.2' x2='764.2' y1='16.2' y2='368.0'/><text class='sa-text' x='764.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='280.1' y2='280.1'/><text class='sa-text' x='57.9' y='286.1' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='208.1' y2='208.1'/><text class='sa-text' x='57.9' y='214.1' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='136.2' y2='136.2'/><text class='sa-text' x='57.9' y='142.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='64.2' y2='64.2'/><text class='sa-text' x='57.9' y='70.2' text-anchor='end' font-size='15'>20</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 342.7,84.9 521.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='521.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='529.2' cy='51.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='697.9' cy='55.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='73.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='342.7' cy='84.9' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='514.8' cy='93.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='802.2' cy='97.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='478.7' cy='99.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='473.1' cy='136.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='490.5' cy='139.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='154.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='744.6' cy='235.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='811.9' cy='243.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='521.5' y1='32.2' x2='401.7' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='401.7' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='529.2' y1='51.4' x2='476.9' y2='59.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='476.9' y='59.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='697.9' y1='55.2' x2='755.7' y2='40.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='755.7' y='40.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='695.4' y1='73.4' x2='659.9' y2='87.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='659.9' y='87.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='342.7' y1='84.9' x2='303.0' y2='99.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='303.0' y='99.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='514.8' y1='93.9' x2='554.5' y2='79.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='554.5' y='79.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><text class='sa-dim sa-halo' x='792.5' y='113.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='478.7' y1='99.5' x2='437.1' y2='112.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='437.1' y='112.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='473.1' y1='136.8' x2='396.9' y2='141.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='396.9' y='141.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='490.5' y1='139.5' x2='584.9' y2='180.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='584.9' y='180.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='812.3' y1='154.2' x2='828.6' y2='167.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='167.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='744.6' y1='235.7' x2='767.9' y2='221.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='767.9' y='221.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='257.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='521.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 22.2% Selection time per 1k features: 38.5 s Datasets: 71 Next to MI: 20.9%, 45.9 s' aria-label='RFImportance (on the Pareto front) Improvement over Random: 22.2% Selection time per 1k features: 38.5 s Datasets: 71 Next to MI: 20.9%, 45.9 s'/><circle class='sa-hit' cx='529.2' cy='51.4' r='10.5' data-tip='MI Improvement over Random: 20.9% Selection time per 1k features: 45.9 s Datasets: 71 Next to RFImportance: 22.2%, 38.5 s' aria-label='MI Improvement over Random: 20.9% Selection time per 1k features: 45.9 s Datasets: 71 Next to RFImportance: 22.2%, 38.5 s'/><circle class='sa-hit' cx='697.9' cy='55.2' r='10.5' data-tip='ElasticNet Improvement over Random: 20.6% Selection time per 1k features: 2.19e+03 s Datasets: 71 Next to Lasso: 19.4%, 2.07e+03 s' aria-label='ElasticNet Improvement over Random: 20.6% Selection time per 1k features: 2.19e+03 s Datasets: 71 Next to Lasso: 19.4%, 2.07e+03 s'/><circle class='sa-hit' cx='695.4' cy='73.4' r='10.5' data-tip='Lasso Improvement over Random: 19.4% Selection time per 1k features: 2.07e+03 s Datasets: 71 Next to ElasticNet: 20.6%, 2.19e+03 s' aria-label='Lasso Improvement over Random: 19.4% Selection time per 1k features: 2.07e+03 s Datasets: 71 Next to ElasticNet: 20.6%, 2.19e+03 s'/><circle class='sa-hit' cx='342.7' cy='84.9' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 18.6% Selection time per 1k features: 0.639 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 18.6% Selection time per 1k features: 0.639 s Datasets: 71'/><circle class='sa-hit' cx='514.8' cy='93.9' r='10.5' data-tip='mRMR Improvement over Random: 17.9% Selection time per 1k features: 33 s Datasets: 71' aria-label='mRMR Improvement over Random: 17.9% Selection time per 1k features: 33 s Datasets: 71'/><circle class='sa-hit' cx='802.2' cy='97.1' r='10.5' data-tip='(R)ReliefF Improvement over Random: 17.7% Selection time per 1k features: 2.39e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 17.7% Selection time per 1k features: 2.39e+04 s Datasets: 71'/><circle class='sa-hit' cx='478.7' cy='99.5' r='10.5' data-tip='CART Improvement over Random: 17.5% Selection time per 1k features: 14.4 s Datasets: 71' aria-label='CART Improvement over Random: 17.5% Selection time per 1k features: 14.4 s Datasets: 71'/><circle class='sa-hit' cx='473.1' cy='136.8' r='10.5' data-tip='GainRatio Improvement over Random: 15.0% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 14.8%, 18.9 s' aria-label='GainRatio Improvement over Random: 15.0% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 14.8%, 18.9 s'/><circle class='sa-hit' cx='490.5' cy='139.5' r='10.5' data-tip='MarkovBlanket Improvement over Random: 14.8% Selection time per 1k features: 18.9 s Datasets: 71 Next to GainRatio: 15.0%, 12.7 s' aria-label='MarkovBlanket Improvement over Random: 14.8% Selection time per 1k features: 18.9 s Datasets: 71 Next to GainRatio: 15.0%, 12.7 s'/><circle class='sa-hit' cx='812.3' cy='154.2' r='10.5' data-tip='SFS Improvement over Random: 13.8% Selection time per 1k features: 3.01e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 13.8% Selection time per 1k features: 3.01e+04 s Datasets: 71'/><circle class='sa-hit' cx='744.6' cy='235.7' r='10.5' data-tip='LOCO Improvement over Random: 8.1% Selection time per 1k features: 6.39e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 8.1% Selection time per 1k features: 6.39e+03 s Datasets: 71'/><circle class='sa-hit' cx='811.9' cy='243.9' r='10.5' data-tip='SBE Improvement over Random: 7.5% Selection time per 1k features: 2.99e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 7.5% Selection time per 1k features: 2.99e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,22.2269,38.5063,71,True
|
| 37 |
MI,20.8936,45.8704,71,False
|
| 38 |
ElasticNet,20.6265,2189.5545,71,False
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='161.2' x2='161.2' y1='16.2' y2='368.0'/><text class='sa-text' x='161.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='261.7' x2='261.7' y1='16.2' y2='368.0'/><text class='sa-text' x='261.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='362.2' x2='362.2' y1='16.2' y2='368.0'/><text class='sa-text' x='362.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='462.7' x2='462.7' y1='16.2' y2='368.0'/><text class='sa-text' x='462.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='563.2' x2='563.2' y1='16.2' y2='368.0'/><text class='sa-text' x='563.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='663.7' x2='663.7' y1='16.2' y2='368.0'/><text class='sa-text' x='663.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='764.2' x2='764.2' y1='16.2' y2='368.0'/><text class='sa-text' x='764.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='280.1' y2='280.1'/><text class='sa-text' x='57.9' y='286.1' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='208.1' y2='208.1'/><text class='sa-text' x='57.9' y='214.1' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='136.2' y2='136.2'/><text class='sa-text' x='57.9' y='142.2' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='64.2' y2='64.2'/><text class='sa-text' x='57.9' y='70.2' text-anchor='end' font-size='15'>20</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 342.7,84.9 521.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='521.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='529.2' cy='51.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='697.9' cy='55.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='695.4' cy='73.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='342.7' cy='84.9' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='514.8' cy='93.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='802.2' cy='97.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='478.7' cy='99.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='473.1' cy='136.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='490.5' cy='139.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='154.2' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='744.6' cy='235.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='811.9' cy='243.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='521.5' y1='32.2' x2='399.2' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='399.2' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='529.2' y1='51.4' x2='481.9' y2='57.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='481.9' y='57.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='697.9' y1='55.2' x2='755.7' y2='40.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='755.7' y='40.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='695.4' y1='73.4' x2='649.8' y2='82.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='649.8' y='82.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='342.7' y1='84.9' x2='303.0' y2='99.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='303.0' y='99.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='514.8' y1='93.9' x2='556.4' y2='79.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='556.4' y='79.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='802.2' y1='97.1' x2='779.9' y2='116.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='779.9' y='116.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='478.7' y1='99.5' x2='437.8' y2='111.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='437.8' y='111.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='473.1' y1='136.8' x2='397.6' y2='148.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='397.6' y='148.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='490.5' y1='139.5' x2='581.1' y2='177.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='581.1' y='177.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='812.3' y1='154.2' x2='828.6' y2='167.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='167.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='744.6' y1='235.7' x2='767.9' y2='221.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='767.9' y='221.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><text class='sa-dim sa-halo' x='827.3' y='257.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='521.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 22.2% Selection time per 1k features: 38.5 s Datasets: 71 Next to MI: 20.9%, 45.9 s' aria-label='RFImportance (on the Pareto front) Improvement over Random: 22.2% Selection time per 1k features: 38.5 s Datasets: 71 Next to MI: 20.9%, 45.9 s'/><circle class='sa-hit' cx='529.2' cy='51.4' r='10.5' data-tip='MI Improvement over Random: 20.9% Selection time per 1k features: 45.9 s Datasets: 71 Next to RFImportance: 22.2%, 38.5 s' aria-label='MI Improvement over Random: 20.9% Selection time per 1k features: 45.9 s Datasets: 71 Next to RFImportance: 22.2%, 38.5 s'/><circle class='sa-hit' cx='697.9' cy='55.2' r='10.5' data-tip='ElasticNet Improvement over Random: 20.6% Selection time per 1k features: 2.19e+03 s Datasets: 71 Next to Lasso: 19.4%, 2.07e+03 s' aria-label='ElasticNet Improvement over Random: 20.6% Selection time per 1k features: 2.19e+03 s Datasets: 71 Next to Lasso: 19.4%, 2.07e+03 s'/><circle class='sa-hit' cx='695.4' cy='73.4' r='10.5' data-tip='Lasso Improvement over Random: 19.4% Selection time per 1k features: 2.07e+03 s Datasets: 71 Next to ElasticNet: 20.6%, 2.19e+03 s' aria-label='Lasso Improvement over Random: 19.4% Selection time per 1k features: 2.07e+03 s Datasets: 71 Next to ElasticNet: 20.6%, 2.19e+03 s'/><circle class='sa-hit' cx='342.7' cy='84.9' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 18.6% Selection time per 1k features: 0.639 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 18.6% Selection time per 1k features: 0.639 s Datasets: 71'/><circle class='sa-hit' cx='514.8' cy='93.9' r='10.5' data-tip='mRMR Improvement over Random: 17.9% Selection time per 1k features: 33 s Datasets: 71' aria-label='mRMR Improvement over Random: 17.9% Selection time per 1k features: 33 s Datasets: 71'/><circle class='sa-hit' cx='802.2' cy='97.1' r='10.5' data-tip='(R)ReliefF Improvement over Random: 17.7% Selection time per 1k features: 2.39e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 17.7% Selection time per 1k features: 2.39e+04 s Datasets: 71'/><circle class='sa-hit' cx='478.7' cy='99.5' r='10.5' data-tip='CART Improvement over Random: 17.5% Selection time per 1k features: 14.4 s Datasets: 71' aria-label='CART Improvement over Random: 17.5% Selection time per 1k features: 14.4 s Datasets: 71'/><circle class='sa-hit' cx='473.1' cy='136.8' r='10.5' data-tip='GainRatio Improvement over Random: 15.0% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 14.8%, 18.9 s' aria-label='GainRatio Improvement over Random: 15.0% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 14.8%, 18.9 s'/><circle class='sa-hit' cx='490.5' cy='139.5' r='10.5' data-tip='MarkovBlanket Improvement over Random: 14.8% Selection time per 1k features: 18.9 s Datasets: 71 Next to GainRatio: 15.0%, 12.7 s' aria-label='MarkovBlanket Improvement over Random: 14.8% Selection time per 1k features: 18.9 s Datasets: 71 Next to GainRatio: 15.0%, 12.7 s'/><circle class='sa-hit' cx='812.3' cy='154.2' r='10.5' data-tip='SFS Improvement over Random: 13.8% Selection time per 1k features: 3.01e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 13.8% Selection time per 1k features: 3.01e+04 s Datasets: 71'/><circle class='sa-hit' cx='744.6' cy='235.7' r='10.5' data-tip='LOCO Improvement over Random: 8.1% Selection time per 1k features: 6.39e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 8.1% Selection time per 1k features: 6.39e+03 s Datasets: 71'/><circle class='sa-hit' cx='811.9' cy='243.9' r='10.5' data-tip='SBE Improvement over Random: 7.5% Selection time per 1k features: 2.99e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 7.5% Selection time per 1k features: 2.99e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,22.2269,38.5063,71,True
|
| 37 |
MI,20.8936,45.8704,71,False
|
| 38 |
ElasticNet,20.6265,2189.5545,71,False
|
data/subsets/tasks_all/models_linearmodel/budgets_2/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,ElasticNet,Non-search,Regularization,1313.0,1257.0,1376.0,5.75,66.1,14.0,,0.481,0.777,6.3,0.0,71,213
|
| 3 |
-
2,Lasso,Non-search,Regularization,1274.0,1219.0,1331.0,6.11,63.5,14.2,,0.389,0.73,1.8,0.0,71,213
|
| 4 |
-
3,RFImportance,Non-search,Tree,1263.0,1213.0,1321.0,6.51,60.7,14.9,,0.676,0.691,0.2,0.0,71,213
|
| 5 |
-
4,MI,Non-search,Information theory,1258.0,1213.0,1303.0,6.65,59.6,16.8,,0.683,0.815,0.4,0.0,71,213
|
| 6 |
-
5,F-Test,Non-search,Correlation,1216.0,1171.0,1262.0,7.2,55.7,17.6,,0.706,0.879,0.0,0.0,71,213
|
| 7 |
-
6,CART,Non-search,Tree,1203.0,1154.0,1258.0,7.48,53.7,19.7,,0.444,0.572,0.1,0.0,71,213
|
| 8 |
-
7,GainRatio,Non-search,Information theory,1185.0,1139.0,1230.0,8.04,49.7,20.8,,0.67,0.856,0.2,0.0,71,213
|
| 9 |
-
8,mRMR,Non-search,Information theory,1173.0,1123.0,1221.0,7.81,51.4,19.1,,0.614,0.66,0.4,0.0,71,213
|
| 10 |
-
9,MarkovBlanket,Search,Backward search,1172.0,1129.0,1214.0,8.03,49.8,21.5,,0.413,0.447,0.1,0.0,71,213
|
| 11 |
-
10,SFS,Search,Forward search,1172.0,1126.0,1215.0,7.82,51.3,20.7,,0.239,0.627,2264.7,47.9,71,213
|
| 12 |
-
11,(R)ReliefF,Non-search,Distance,1168.0,1123.0,1212.0,8.41,47.1,19.5,,0.673,0.757,12.8,0.0,71,213
|
| 13 |
-
12,LOCO,Search,Leave-one-out,1109.0,1061.0,1150.0,8.81,44.2,25.7,,0.151,0.575,242.0,6.1,71,213
|
| 14 |
-
13,SBE,Search,Backward search,1088.0,1039.0,1135.0,8.94,43.3,25.1,,0.144,0.556,2787.0,48.4,71,213
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,935.0,1056.0,10.69,30.8,31.5,,-0.001,0.513,0.0,0.0,71,213
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,942.0,868.0,997.0,11.74,23.3,37.1,,0.66,0.827,0.2,0.0,71,213
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,ElasticNet,Non-search,Regularization,1313.0,1257.0,1376.0,5.75,66.1,14.0,20.6,,0.481,0.777,6.3,0.0,71,213
|
| 3 |
+
2,Lasso,Non-search,Regularization,1274.0,1219.0,1331.0,6.11,63.5,14.2,19.4,,0.389,0.73,1.8,0.0,71,213
|
| 4 |
+
3,RFImportance,Non-search,Tree,1263.0,1213.0,1321.0,6.51,60.7,14.9,22.2,,0.676,0.691,0.2,0.0,71,213
|
| 5 |
+
4,MI,Non-search,Information theory,1258.0,1213.0,1303.0,6.65,59.6,16.8,20.9,,0.683,0.815,0.4,0.0,71,213
|
| 6 |
+
5,F-Test,Non-search,Correlation,1216.0,1171.0,1262.0,7.2,55.7,17.6,18.6,,0.706,0.879,0.0,0.0,71,213
|
| 7 |
+
6,CART,Non-search,Tree,1203.0,1154.0,1258.0,7.48,53.7,19.7,17.5,,0.444,0.572,0.1,0.0,71,213
|
| 8 |
+
7,GainRatio,Non-search,Information theory,1185.0,1139.0,1230.0,8.04,49.7,20.8,15.0,,0.67,0.856,0.2,0.0,71,213
|
| 9 |
+
8,mRMR,Non-search,Information theory,1173.0,1123.0,1221.0,7.81,51.4,19.1,17.9,,0.614,0.66,0.4,0.0,71,213
|
| 10 |
+
9,MarkovBlanket,Search,Backward search,1172.0,1129.0,1214.0,8.03,49.8,21.5,14.8,,0.413,0.447,0.1,0.0,71,213
|
| 11 |
+
10,SFS,Search,Forward search,1172.0,1126.0,1215.0,7.82,51.3,20.7,13.8,,0.239,0.627,2264.7,47.9,71,213
|
| 12 |
+
11,(R)ReliefF,Non-search,Distance,1168.0,1123.0,1212.0,8.41,47.1,19.5,17.7,,0.673,0.757,12.8,0.0,71,213
|
| 13 |
+
12,LOCO,Search,Leave-one-out,1109.0,1061.0,1150.0,8.81,44.2,25.7,8.1,,0.151,0.575,242.0,6.1,71,213
|
| 14 |
+
13,SBE,Search,Backward search,1088.0,1039.0,1135.0,8.94,43.3,25.1,7.5,,0.144,0.556,2787.0,48.4,71,213
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,935.0,1056.0,10.69,30.8,31.5,0.0,,-0.001,0.513,0.0,0.0,71,213
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,942.0,868.0,997.0,11.74,23.3,37.1,-52.8,,0.66,0.827,0.2,0.0,71,213
|
data/subsets/tasks_all/models_linearmodel/budgets_3/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1246.0'>1246<span class='sub'>+54 / −51</span></td><td class='num' data-sort='6.28'>6.28</td><td class='num' data-sort='12.0'>12.0</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1237.0'>1237<span class='sub'>+55 / −48</span></td><td class='num' data-sort='6.43'>6.43</td><td class='num' data-sort='12.7'>12.7</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1223.0'>1223<span class='sub'>+63 / −54</span></td><td class='num' data-sort='6.65'>6.65</td><td class='num' data-sort='13.6'>13.6</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1213.0'>1213<span class='sub'>+49 / −47</span></td><td class='num' data-sort='6.88'>6.88</td><td class='num' data-sort='14.5'>14.5</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+45 / −46</span></td><td class='num' data-sort='7.21'>7.21</td><td class='num' data-sort='15.8'>15.8</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1162.0'>1162<span class='sub'>+49 / −44</span></td><td class='num' data-sort='7.32'>7.32</td><td class='num' data-sort='16.0'>16.0</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1152.0'>1152<span class='sub'>+47 / −47</span></td><td class='num' data-sort='7.7'>7.70</td><td class='num' data-sort='15.6'>15.6</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1130.0'>1130<span class='sub'>+45 / −50</span></td><td class='num' data-sort='7.96'>7.96</td><td class='num' data-sort='17.8'>17.8</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2680.3'>2,680.3</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1124.0'>1124<span class='sub'>+45 / −40</span></td><td class='num' data-sort='8.39'>8.39</td><td class='num' data-sort='16.5'>16.5</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1121.0'>1121<span class='sub'>+49 / −57</span></td><td class='num' data-sort='8.51'>8.51</td><td class='num' data-sort='19.3'>19.3</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1866.7'>1,866.7</td><td class='num' data-sort='45.1'>45.1</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1119.0'>1119<span class='sub'>+46 / −43</span></td><td class='num' data-sort='8.55'>8.55</td><td class='num' data-sort='17.6'>17.6</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1108.0'>1108<span class='sub'>+42 / −46</span></td><td class='num' data-sort='8.34'>8.34</td><td class='num' data-sort='19.1'>19.1</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='246.0'>246.0</td><td class='num' data-sort='11.7'>11.7</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1106.0'>1106<span class='sub'>+46 / −48</span></td><td class='num' data-sort='8.73'>8.73</td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+47 / −53</span></td><td class='num' data-sort='9.98'>9.98</td><td class='num' data-sort='23.7'>23.7</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='977.0'>977<span class='sub'>+58 / −69</span></td><td class='num' data-sort='11.07'>11.07</td><td class='num' data-sort='29.8'>29.8</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1246.0,1195.0,1300.0,
|
| 37 |
-
2,ElasticNet,Non-search,Regularization,1237.0,1189.0,1292.0,
|
| 38 |
-
3,Lasso,Non-search,Regularization,1223.0,1169.0,1286.0,
|
| 39 |
-
4,MI,Non-search,Information theory,1213.0,1166.0,1262.0,
|
| 40 |
-
5,F-Test,Non-search,Correlation,1186.0,1140.0,1231.0,
|
| 41 |
-
6,CART,Non-search,Tree,1162.0,1118.0,1211.0,
|
| 42 |
-
7,mRMR,Non-search,Information theory,1152.0,1105.0,1199.0,
|
| 43 |
-
8,SFS,Search,Forward search,1130.0,1080.0,1175.0,7.
|
| 44 |
-
9,(R)ReliefF,Non-search,Distance,1124.0,1084.0,1169.0,
|
| 45 |
-
10,SBE,Search,Backward search,1121.0,1064.0,1170.0,
|
| 46 |
-
11,GainRatio,Non-search,Information theory,1119.0,1076.0,1165.0,
|
| 47 |
-
12,LOCO,Search,Leave-one-out,1108.0,1062.0,1150.0,
|
| 48 |
-
13,MarkovBlanket,Search,Backward search,1106.0,1058.0,1152.0,
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,947.0,1047.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,977.0,908.0,1035.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1246.0'>1246<span class='sub'>+54 / −51</span></td><td class='num' data-sort='15.9'>15.9</td><td class='num shaded' data-sort='0.661' style='background:rgba(0,114,178,0.52)'>0.661</td><td class='num shaded' data-sort='0.6' style='background:rgba(213,94,0,0.48)'>0.600</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1237.0'>1237<span class='sub'>+55 / −48</span></td><td class='num' data-sort='13.7'>13.7</td><td class='num shaded' data-sort='0.419' style='background:rgba(0,114,178,0.36)'>0.419</td><td class='num shaded' data-sort='0.665' style='background:rgba(213,94,0,0.53)'>0.665</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1223.0'>1223<span class='sub'>+63 / −54</span></td><td class='num' data-sort='12.2'>12.2</td><td class='num shaded' data-sort='0.334' style='background:rgba(0,114,178,0.30)'>0.334</td><td class='num shaded' data-sort='0.643' style='background:rgba(213,94,0,0.51)'>0.643</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1213.0'>1213<span class='sub'>+49 / −47</span></td><td class='num' data-sort='12.5'>12.5</td><td class='num shaded' data-sort='0.649' style='background:rgba(0,114,178,0.51)'>0.649</td><td class='num shaded' data-sort='0.723' style='background:rgba(213,94,0,0.56)'>0.723</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+45 / −46</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.668' style='background:rgba(0,114,178,0.53)'>0.668</td><td class='num shaded' data-sort='0.781' style='background:rgba(213,94,0,0.60)'>0.781</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1162.0'>1162<span class='sub'>+49 / −44</span></td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.402' style='background:rgba(0,114,178,0.35)'>0.402</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1152.0'>1152<span class='sub'>+47 / −47</span></td><td class='num' data-sort='11.8'>11.8</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.593' style='background:rgba(213,94,0,0.48)'>0.593</td><td class='num' data-sort='0.5'>0.5</td><td class='num' data-sort='7.0'>7.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1130.0'>1130<span class='sub'>+45 / −50</span></td><td class='num' data-sort='7.8'>7.8</td><td class='num shaded' data-sort='0.225' style='background:rgba(0,114,178,0.23)'>0.225</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='2680.3'>2,680.3</td><td class='num' data-sort='48.4'>48.4</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1124.0'>1124<span class='sub'>+45 / −40</span></td><td class='num' data-sort='9.2'>9.2</td><td class='num shaded' data-sort='0.656' style='background:rgba(0,114,178,0.52)'>0.656</td><td class='num shaded' data-sort='0.667' style='background:rgba(213,94,0,0.53)'>0.667</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1121.0'>1121<span class='sub'>+49 / −57</span></td><td class='num' data-sort='5.8'>5.8</td><td class='num shaded' data-sort='0.095' style='background:rgba(0,114,178,0.14)'>0.095</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='1866.7'>1,866.7</td><td class='num' data-sort='45.1'>45.1</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1119.0'>1119<span class='sub'>+46 / −43</span></td><td class='num' data-sort='9.2'>9.2</td><td class='num shaded' data-sort='0.641' style='background:rgba(0,114,178,0.51)'>0.641</td><td class='num shaded' data-sort='0.751' style='background:rgba(213,94,0,0.58)'>0.751</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1108.0'>1108<span class='sub'>+42 / −46</span></td><td class='num' data-sort='6.5'>6.5</td><td class='num shaded' data-sort='0.114' style='background:rgba(0,114,178,0.16)'>0.114</td><td class='num shaded' data-sort='0.544' style='background:rgba(213,94,0,0.44)'>0.544</td><td class='num' data-sort='246.0'>246.0</td><td class='num' data-sort='11.7'>11.7</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1106.0'>1106<span class='sub'>+46 / −48</span></td><td class='num' data-sort='7.8'>7.8</td><td class='num shaded' data-sort='0.434' style='background:rgba(0,114,178,0.37)'>0.434</td><td class='num shaded' data-sort='0.473' style='background:rgba(213,94,0,0.40)'>0.473</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+47 / −53</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.494' style='background:rgba(213,94,0,0.41)'>0.494</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='977.0'>977<span class='sub'>+58 / −69</span></td><td class='num' data-sort='-21.5'>-21.5</td><td class='num shaded' data-sort='0.689' style='background:rgba(0,114,178,0.54)'>0.689</td><td class='num shaded' data-sort='0.758' style='background:rgba(213,94,0,0.59)'>0.758</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1246.0,1195.0,1300.0,15.9,0.661,0.6,0.2,0.0
|
| 37 |
+
2,ElasticNet,Non-search,Regularization,1237.0,1189.0,1292.0,13.7,0.419,0.665,6.2,0.0
|
| 38 |
+
3,Lasso,Non-search,Regularization,1223.0,1169.0,1286.0,12.2,0.334,0.643,1.9,0.0
|
| 39 |
+
4,MI,Non-search,Information theory,1213.0,1166.0,1262.0,12.5,0.649,0.723,0.4,0.0
|
| 40 |
+
5,F-Test,Non-search,Correlation,1186.0,1140.0,1231.0,11.1,0.668,0.781,0.0,0.0
|
| 41 |
+
6,CART,Non-search,Tree,1162.0,1118.0,1211.0,11.1,0.402,0.507,0.1,0.0
|
| 42 |
+
7,mRMR,Non-search,Information theory,1152.0,1105.0,1199.0,11.8,0.656,0.593,0.5,7.0
|
| 43 |
+
8,SFS,Search,Forward search,1130.0,1080.0,1175.0,7.8,0.225,0.568,2680.3,48.4
|
| 44 |
+
9,(R)ReliefF,Non-search,Distance,1124.0,1084.0,1169.0,9.2,0.656,0.667,13.2,0.0
|
| 45 |
+
10,SBE,Search,Backward search,1121.0,1064.0,1170.0,5.8,0.095,0.53,1866.7,45.1
|
| 46 |
+
11,GainRatio,Non-search,Information theory,1119.0,1076.0,1165.0,9.2,0.641,0.751,0.2,0.0
|
| 47 |
+
12,LOCO,Search,Leave-one-out,1108.0,1062.0,1150.0,6.5,0.114,0.544,246.0,11.7
|
| 48 |
+
13,MarkovBlanket,Search,Backward search,1106.0,1058.0,1152.0,7.8,0.434,0.473,0.1,0.0
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,947.0,1047.0,0.0,0.002,0.494,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,977.0,908.0,1035.0,-21.5,0.689,0.758,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_linearmodel/budgets_3/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='160.8' x2='160.8' y1='16.2' y2='368.0'/><text class='sa-text' x='160.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='260.5' x2='260.5' y1='16.2' y2='368.0'/><text class='sa-text' x='260.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='360.3' x2='360.3' y1='16.2' y2='368.0'/><text class='sa-text' x='360.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='460.1' x2='460.1' y1='16.2' y2='368.0'/><text class='sa-text' x='460.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.8' x2='559.8' y1='16.2' y2='368.0'/><text class='sa-text' x='559.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='659.6' x2='659.6' y1='16.2' y2='368.0'/><text class='sa-text' x='659.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='759.4' x2='759.4' y1='16.2' y2='368.0'/><text class='sa-text' x='759.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='311.7' y2='311.7'/><text class='sa-text' x='57.9' y='317.7' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='271.4' y2='271.4'/><text class='sa-text' x='57.9' y='277.4' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='231.0' y2='231.0'/><text class='sa-text' x='57.9' y='237.0' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='190.7' y2='190.7'/><text class='sa-text' x='57.9' y='196.7' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='150.4' y2='150.4'/><text class='sa-text' x='57.9' y='156.4' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='110.0' y2='110.0'/><text class='sa-text' x='57.9' y='116.0' text-anchor='end' font-size='15'>12</text><line class='sa-grid' x1='66.9' x2='847.8' y1='69.7' y2='69.7'/><text class='sa-text' x='57.9' y='75.7' text-anchor='end' font-size='15'>14</text><line class='sa-grid' x1='66.9' x2='847.8' y1='29.3' y2='29.3'/><text class='sa-text' x='57.9' y='35.3' text-anchor='end' font-size='15'>16</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 343.1,127.5 520.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='520.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='691.8' cy='76.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='526.6' cy='100.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='690.9' cy='106.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='540.1' cy='114.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='343.1' cy='127.5' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='476.8' cy='128.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='470.6' cy='165.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.8' cy='167.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='195.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='486.2' cy='195.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='740.3' cy='220.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.4' cy='234.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='520.8' y1='32.2' x2='398.0' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='398.0' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='691.8' y1='76.7' x2='743.6' y2='62.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='743.6' y='62.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='541.1' y='86.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='690.9' y1='106.5' x2='663.7' y2='119.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='663.7' y='119.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='540.1' y1='114.1' x2='556.5' y2='130.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='556.5' y='130.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='343.1' y1='127.5' x2='302.1' y2='141.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='302.1' y='141.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='476.8' y1='128.8' x2='440.9' y2='143.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='440.9' y='143.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='470.6' y1='165.9' x2='406.4' y2='179.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='406.4' y='179.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='796.9' y='153.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='195.4' x2='828.6' y2='209.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='209.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='486.2' y1='195.7' x2='590.7' y2='210.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='590.7' y='210.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='740.3' y1='220.0' x2='756.0' y2='205.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='756.0' y='205.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='797.4' y1='234.4' x2='823.9' y2='248.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='823.9' y='248.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='175.1' y2='358.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='175.1' y='358.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='520.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 15.9% Selection time per 1k features: 40.6 s Datasets: 70' aria-label='RFImportance (on the Pareto front) Improvement over Random: 15.9% Selection time per 1k features: 40.6 s Datasets: 70'/><circle class='sa-hit' cx='691.8' cy='76.7' r='10.5' data-tip='ElasticNet Improvement over Random: 13.7% Selection time per 1k features: 2.1e+03 s Datasets: 70' aria-label='ElasticNet Improvement over Random: 13.7% Selection time per 1k features: 2.1e+03 s Datasets: 70'/><circle class='sa-hit' cx='526.6' cy='100.9' r='10.5' data-tip='MI Improvement over Random: 12.5% Selection time per 1k features: 46.4 s Datasets: 70 Next to mRMR: 11.8%, 63.4 s' aria-label='MI Improvement over Random: 12.5% Selection time per 1k features: 46.4 s Datasets: 70 Next to mRMR: 11.8%, 63.4 s'/><circle class='sa-hit' cx='690.9' cy='106.5' r='10.5' data-tip='Lasso Improvement over Random: 12.2% Selection time per 1k features: 2.06e+03 s Datasets: 70' aria-label='Lasso Improvement over Random: 12.2% Selection time per 1k features: 2.06e+03 s Datasets: 70'/><circle class='sa-hit' cx='540.1' cy='114.1' r='10.5' data-tip='mRMR Improvement over Random: 11.8% Selection time per 1k features: 63.4 s Datasets: 70 Next to MI: 12.5%, 46.4 s' aria-label='mRMR Improvement over Random: 11.8% Selection time per 1k features: 63.4 s Datasets: 70 Next to MI: 12.5%, 46.4 s'/><circle class='sa-hit' cx='343.1' cy='127.5' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 0.671 s Datasets: 70' aria-label='F-Test (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 0.671 s Datasets: 70'/><circle class='sa-hit' cx='476.8' cy='128.8' r='10.5' data-tip='CART Improvement over Random: 11.1% Selection time per 1k features: 14.7 s Datasets: 70' aria-label='CART Improvement over Random: 11.1% Selection time per 1k features: 14.7 s Datasets: 70'/><circle class='sa-hit' cx='470.6' cy='165.9' r='10.5' data-tip='GainRatio Improvement over Random: 9.2% Selection time per 1k features: 12.8 s Datasets: 70' aria-label='GainRatio Improvement over Random: 9.2% Selection time per 1k features: 12.8 s Datasets: 70'/><circle class='sa-hit' cx='797.8' cy='167.4' r='10.5' data-tip='(R)ReliefF Improvement over Random: 9.2% Selection time per 1k features: 2.42e+04 s Datasets: 70' aria-label='(R)ReliefF Improvement over Random: 9.2% Selection time per 1k features: 2.42e+04 s Datasets: 70'/><circle class='sa-hit' cx='812.3' cy='195.4' r='10.5' data-tip='SFS Improvement over Random: 7.8% Selection time per 1k features: 3.39e+04 s Datasets: 70' aria-label='SFS Improvement over Random: 7.8% Selection time per 1k features: 3.39e+04 s Datasets: 70'/><circle class='sa-hit' cx='486.2' cy='195.7' r='10.5' data-tip='MarkovBlanket Improvement over Random: 7.8% Selection time per 1k features: 18.3 s Datasets: 70' aria-label='MarkovBlanket Improvement over Random: 7.8% Selection time per 1k features: 18.3 s Datasets: 70'/><circle class='sa-hit' cx='740.3' cy='220.0' r='10.5' data-tip='LOCO Improvement over Random: 6.5% Selection time per 1k features: 6.43e+03 s Datasets: 70' aria-label='LOCO Improvement over Random: 6.5% Selection time per 1k features: 6.43e+03 s Datasets: 70'/><circle class='sa-hit' cx='797.4' cy='234.4' r='10.5' data-tip='SBE Improvement over Random: 5.8% Selection time per 1k features: 2.41e+04 s Datasets: 70' aria-label='SBE Improvement over Random: 5.8% Selection time per 1k features: 2.41e+04 s Datasets: 70'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 70' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 70'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,15.8586,40.579,70,True
|
| 37 |
ElasticNet,13.651,2100.4728,70,False
|
| 38 |
MI,12.4516,46.4375,70,False
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='160.8' x2='160.8' y1='16.2' y2='368.0'/><text class='sa-text' x='160.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='260.5' x2='260.5' y1='16.2' y2='368.0'/><text class='sa-text' x='260.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='360.3' x2='360.3' y1='16.2' y2='368.0'/><text class='sa-text' x='360.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='460.1' x2='460.1' y1='16.2' y2='368.0'/><text class='sa-text' x='460.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='559.8' x2='559.8' y1='16.2' y2='368.0'/><text class='sa-text' x='559.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='659.6' x2='659.6' y1='16.2' y2='368.0'/><text class='sa-text' x='659.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='759.4' x2='759.4' y1='16.2' y2='368.0'/><text class='sa-text' x='759.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='311.7' y2='311.7'/><text class='sa-text' x='57.9' y='317.7' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='271.4' y2='271.4'/><text class='sa-text' x='57.9' y='277.4' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='231.0' y2='231.0'/><text class='sa-text' x='57.9' y='237.0' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='190.7' y2='190.7'/><text class='sa-text' x='57.9' y='196.7' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='150.4' y2='150.4'/><text class='sa-text' x='57.9' y='156.4' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='110.0' y2='110.0'/><text class='sa-text' x='57.9' y='116.0' text-anchor='end' font-size='15'>12</text><line class='sa-grid' x1='66.9' x2='847.8' y1='69.7' y2='69.7'/><text class='sa-text' x='57.9' y='75.7' text-anchor='end' font-size='15'>14</text><line class='sa-grid' x1='66.9' x2='847.8' y1='29.3' y2='29.3'/><text class='sa-text' x='57.9' y='35.3' text-anchor='end' font-size='15'>16</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 343.1,127.5 520.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='520.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='691.8' cy='76.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='526.6' cy='100.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='690.9' cy='106.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='540.1' cy='114.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='343.1' cy='127.5' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='476.8' cy='128.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='470.6' cy='165.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.8' cy='167.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='195.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='486.2' cy='195.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='740.3' cy='220.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='797.4' cy='234.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='520.8' y1='32.2' x2='396.1' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='396.1' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='691.8' y1='76.7' x2='743.6' y2='62.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='743.6' y='62.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='541.1' y='86.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='690.9' y1='106.5' x2='663.7' y2='119.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='663.7' y='119.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='540.1' y1='114.1' x2='556.5' y2='130.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='556.5' y='130.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='343.1' y1='127.5' x2='302.1' y2='141.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='302.1' y='141.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='476.8' y1='128.8' x2='440.9' y2='143.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='440.9' y='143.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='470.6' y1='165.9' x2='406.4' y2='179.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='406.4' y='179.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='796.9' y='153.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='195.4' x2='828.6' y2='209.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='209.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='486.2' y1='195.7' x2='592.0' y2='209.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='592.0' y='209.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><text class='sa-dim sa-halo' x='754.8' y='205.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='797.4' y1='234.4' x2='823.9' y2='248.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='823.9' y='248.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='175.1' y2='358.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='175.1' y='358.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='520.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 15.9% Selection time per 1k features: 40.6 s Datasets: 70' aria-label='RFImportance (on the Pareto front) Improvement over Random: 15.9% Selection time per 1k features: 40.6 s Datasets: 70'/><circle class='sa-hit' cx='691.8' cy='76.7' r='10.5' data-tip='ElasticNet Improvement over Random: 13.7% Selection time per 1k features: 2.1e+03 s Datasets: 70' aria-label='ElasticNet Improvement over Random: 13.7% Selection time per 1k features: 2.1e+03 s Datasets: 70'/><circle class='sa-hit' cx='526.6' cy='100.9' r='10.5' data-tip='MI Improvement over Random: 12.5% Selection time per 1k features: 46.4 s Datasets: 70 Next to mRMR: 11.8%, 63.4 s' aria-label='MI Improvement over Random: 12.5% Selection time per 1k features: 46.4 s Datasets: 70 Next to mRMR: 11.8%, 63.4 s'/><circle class='sa-hit' cx='690.9' cy='106.5' r='10.5' data-tip='Lasso Improvement over Random: 12.2% Selection time per 1k features: 2.06e+03 s Datasets: 70' aria-label='Lasso Improvement over Random: 12.2% Selection time per 1k features: 2.06e+03 s Datasets: 70'/><circle class='sa-hit' cx='540.1' cy='114.1' r='10.5' data-tip='mRMR Improvement over Random: 11.8% Selection time per 1k features: 63.4 s Datasets: 70 Next to MI: 12.5%, 46.4 s' aria-label='mRMR Improvement over Random: 11.8% Selection time per 1k features: 63.4 s Datasets: 70 Next to MI: 12.5%, 46.4 s'/><circle class='sa-hit' cx='343.1' cy='127.5' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 0.671 s Datasets: 70' aria-label='F-Test (on the Pareto front) Improvement over Random: 11.1% Selection time per 1k features: 0.671 s Datasets: 70'/><circle class='sa-hit' cx='476.8' cy='128.8' r='10.5' data-tip='CART Improvement over Random: 11.1% Selection time per 1k features: 14.7 s Datasets: 70' aria-label='CART Improvement over Random: 11.1% Selection time per 1k features: 14.7 s Datasets: 70'/><circle class='sa-hit' cx='470.6' cy='165.9' r='10.5' data-tip='GainRatio Improvement over Random: 9.2% Selection time per 1k features: 12.8 s Datasets: 70' aria-label='GainRatio Improvement over Random: 9.2% Selection time per 1k features: 12.8 s Datasets: 70'/><circle class='sa-hit' cx='797.8' cy='167.4' r='10.5' data-tip='(R)ReliefF Improvement over Random: 9.2% Selection time per 1k features: 2.42e+04 s Datasets: 70' aria-label='(R)ReliefF Improvement over Random: 9.2% Selection time per 1k features: 2.42e+04 s Datasets: 70'/><circle class='sa-hit' cx='812.3' cy='195.4' r='10.5' data-tip='SFS Improvement over Random: 7.8% Selection time per 1k features: 3.39e+04 s Datasets: 70' aria-label='SFS Improvement over Random: 7.8% Selection time per 1k features: 3.39e+04 s Datasets: 70'/><circle class='sa-hit' cx='486.2' cy='195.7' r='10.5' data-tip='MarkovBlanket Improvement over Random: 7.8% Selection time per 1k features: 18.3 s Datasets: 70' aria-label='MarkovBlanket Improvement over Random: 7.8% Selection time per 1k features: 18.3 s Datasets: 70'/><circle class='sa-hit' cx='740.3' cy='220.0' r='10.5' data-tip='LOCO Improvement over Random: 6.5% Selection time per 1k features: 6.43e+03 s Datasets: 70' aria-label='LOCO Improvement over Random: 6.5% Selection time per 1k features: 6.43e+03 s Datasets: 70'/><circle class='sa-hit' cx='797.4' cy='234.4' r='10.5' data-tip='SBE Improvement over Random: 5.8% Selection time per 1k features: 2.41e+04 s Datasets: 70' aria-label='SBE Improvement over Random: 5.8% Selection time per 1k features: 2.41e+04 s Datasets: 70'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 70' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 70'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,15.8586,40.579,70,True
|
| 37 |
ElasticNet,13.651,2100.4728,70,False
|
| 38 |
MI,12.4516,46.4375,70,False
|
data/subsets/tasks_all/models_linearmodel/budgets_3/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1246.0,1195.0,1300.0,6.28,62.3,12.0,,0.661,0.6,0.2,0.0,71,213
|
| 3 |
-
2,ElasticNet,Non-search,Regularization,1237.0,1189.0,1292.0,6.43,61.2,12.7,,0.419,0.665,6.2,0.0,71,213
|
| 4 |
-
3,Lasso,Non-search,Regularization,1223.0,1169.0,1286.0,6.65,59.7,13.6,,0.334,0.643,1.9,0.0,71,213
|
| 5 |
-
4,MI,Non-search,Information theory,1213.0,1166.0,1262.0,6.88,58.0,14.5,,0.649,0.723,0.4,0.0,71,213
|
| 6 |
-
5,F-Test,Non-search,Correlation,1186.0,1140.0,1231.0,7.21,55.6,15.8,,0.668,0.781,0.0,0.0,71,213
|
| 7 |
-
6,CART,Non-search,Tree,1162.0,1118.0,1211.0,7.32,54.8,16.0,,0.402,0.507,0.1,0.0,71,213
|
| 8 |
-
7,mRMR,Non-search,Information theory,1152.0,1105.0,1199.0,7.7,52.1,15.6,,0.656,0.593,0.5,7.0,71,213
|
| 9 |
-
8,SFS,Search,Forward search,1130.0,1080.0,1175.0,7.96,50.3,17.8,,0.225,0.568,2680.3,48.4,71,213
|
| 10 |
-
9,(R)ReliefF,Non-search,Distance,1124.0,1084.0,1169.0,8.39,47.2,16.5,,0.656,0.667,13.2,0.0,71,213
|
| 11 |
-
10,SBE,Search,Backward search,1121.0,1064.0,1170.0,8.51,46.4,19.3,,0.095,0.53,1866.7,45.1,71,213
|
| 12 |
-
11,GainRatio,Non-search,Information theory,1119.0,1076.0,1165.0,8.55,46.1,17.6,,0.641,0.751,0.2,0.0,71,213
|
| 13 |
-
12,LOCO,Search,Leave-one-out,1108.0,1062.0,1150.0,8.34,47.6,19.1,,0.114,0.544,246.0,11.7,71,213
|
| 14 |
-
13,MarkovBlanket,Search,Backward search,1106.0,1058.0,1152.0,8.73,44.8,19.0,,0.434,0.473,0.1,0.0,71,213
|
| 15 |
-
14,Random,Baseline,Baseline,1000.0,947.0,1047.0,9.98,35.8,23.7,,0.002,0.494,0.0,0.0,71,213
|
| 16 |
-
15,LaplacianScore,Non-search,Distance,977.0,908.0,1035.0,11.07,28.1,29.8,,0.689,0.758,0.2,0.0,71,213
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1246.0,1195.0,1300.0,6.28,62.3,12.0,15.9,,0.661,0.6,0.2,0.0,71,213
|
| 3 |
+
2,ElasticNet,Non-search,Regularization,1237.0,1189.0,1292.0,6.43,61.2,12.7,13.7,,0.419,0.665,6.2,0.0,71,213
|
| 4 |
+
3,Lasso,Non-search,Regularization,1223.0,1169.0,1286.0,6.65,59.7,13.6,12.2,,0.334,0.643,1.9,0.0,71,213
|
| 5 |
+
4,MI,Non-search,Information theory,1213.0,1166.0,1262.0,6.88,58.0,14.5,12.5,,0.649,0.723,0.4,0.0,71,213
|
| 6 |
+
5,F-Test,Non-search,Correlation,1186.0,1140.0,1231.0,7.21,55.6,15.8,11.1,,0.668,0.781,0.0,0.0,71,213
|
| 7 |
+
6,CART,Non-search,Tree,1162.0,1118.0,1211.0,7.32,54.8,16.0,11.1,,0.402,0.507,0.1,0.0,71,213
|
| 8 |
+
7,mRMR,Non-search,Information theory,1152.0,1105.0,1199.0,7.7,52.1,15.6,11.8,,0.656,0.593,0.5,7.0,71,213
|
| 9 |
+
8,SFS,Search,Forward search,1130.0,1080.0,1175.0,7.96,50.3,17.8,7.8,,0.225,0.568,2680.3,48.4,71,213
|
| 10 |
+
9,(R)ReliefF,Non-search,Distance,1124.0,1084.0,1169.0,8.39,47.2,16.5,9.2,,0.656,0.667,13.2,0.0,71,213
|
| 11 |
+
10,SBE,Search,Backward search,1121.0,1064.0,1170.0,8.51,46.4,19.3,5.8,,0.095,0.53,1866.7,45.1,71,213
|
| 12 |
+
11,GainRatio,Non-search,Information theory,1119.0,1076.0,1165.0,8.55,46.1,17.6,9.2,,0.641,0.751,0.2,0.0,71,213
|
| 13 |
+
12,LOCO,Search,Leave-one-out,1108.0,1062.0,1150.0,8.34,47.6,19.1,6.5,,0.114,0.544,246.0,11.7,71,213
|
| 14 |
+
13,MarkovBlanket,Search,Backward search,1106.0,1058.0,1152.0,8.73,44.8,19.0,7.8,,0.434,0.473,0.1,0.0,71,213
|
| 15 |
+
14,Random,Baseline,Baseline,1000.0,947.0,1047.0,9.98,35.8,23.7,0.0,,0.002,0.494,0.0,0.0,71,213
|
| 16 |
+
15,LaplacianScore,Non-search,Distance,977.0,908.0,1035.0,11.07,28.1,29.8,-21.5,,0.689,0.758,0.2,0.0,71,213
|
data/subsets/tasks_all/models_linearmodel/budgets_4/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1220.0'>1220<span class='sub'>+53 / −48</span></td><td class='num' data-sort='6.72'>6.72</td><td class='num' data-sort='10.2'>10.2</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+48 / −44</span></td><td class='num' data-sort='7.21'>7.21</td><td class='num' data-sort='11.1'>11.1</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1187.0'>1187<span class='sub'>+48 / −39</span></td><td class='num' data-sort='7.06'>7.06</td><td class='num' data-sort='10.5'>10.5</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1182.0'>1182<span class='sub'>+43 / −42</span></td><td class='num' data-sort='7.14'>7.14</td><td class='num' data-sort='11.0'>11.0</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1176.0'>1176<span class='sub'>+46 / −47</span></td><td class='num' data-sort='7.62'>7.62</td><td class='num' data-sort='10.6'>10.6</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1170.0'>1170<span class='sub'>+54 / −46</span></td><td class='num' data-sort='7.1'>7.10</td><td class='num' data-sort='11.6'>11.6</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+49 / −45</span></td><td class='num' data-sort='7.55'>7.55</td><td class='num' data-sort='12.6'>12.6</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1138.0'>1138<span class='sub'>+52 / −45</span></td><td class='num' data-sort='8.11'>8.11</td><td class='num' data-sort='11.9'>11.9</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='13.4'>13.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1121.0'>1121<span class='sub'>+49 / −49</span></td><td class='num' data-sort='8.42'>8.42</td><td class='num' data-sort='12.9'>12.9</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1114.0'>1114<span class='sub'>+49 / −53</span></td><td class='num' data-sort='8.54'>8.54</td><td class='num' data-sort='12.6'>12.6</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1111.0'>1111<span class='sub'>+45 / −53</span></td><td class='num' data-sort='8.55'>8.55</td><td class='num' data-sort='14.8'>14.8</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='1035.1'>1,035.1</td><td class='num' data-sort='41.3'>41.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1102.0'>1102<span class='sub'>+46 / −46</span></td><td class='num' data-sort='8.47'>8.47</td><td class='num' data-sort='15.2'>15.2</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='3042.8'>3,042.8</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1089.0'>1089<span class='sub'>+45 / −46</span></td><td class='num' data-sort='8.29'>8.29</td><td class='num' data-sort='15.2'>15.2</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='234.7'>234.7</td><td class='num' data-sort='18.3'>18.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1042.0'>1042<span class='sub'>+56 / −66</span></td><td class='num' data-sort='9.77'>9.77</td><td class='num' data-sort='20.5'>20.5</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+52 / −59</span></td><td class='num' data-sort='9.46'>9.46</td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,RFImportance,Non-search,Tree,1220.0,1172.0,1273.0,
|
| 37 |
-
2,F-Test,Non-search,Correlation,1189.0,1145.0,1237.0,
|
| 38 |
-
3,ElasticNet,Non-search,Regularization,1187.0,1148.0,1235.0,
|
| 39 |
-
4,MI,Non-search,Information theory,1182.0,1140.0,1225.0,7.
|
| 40 |
-
5,mRMR,Non-search,Information theory,1176.0,1129.0,1222.0,
|
| 41 |
-
6,Lasso,Non-search,Regularization,1170.0,1124.0,1224.0,
|
| 42 |
-
7,CART,Non-search,Tree,1169.0,1124.0,1218.0,
|
| 43 |
-
8,(R)ReliefF,Non-search,Distance,1138.0,1093.0,1190.0,
|
| 44 |
-
9,GainRatio,Non-search,Information theory,1121.0,1072.0,1170.0,
|
| 45 |
-
10,MarkovBlanket,Search,Backward search,1114.0,1061.0,1163.0,
|
| 46 |
-
11,SBE,Search,Backward search,1111.0,1058.0,1156.0,
|
| 47 |
-
12,SFS,Search,Forward search,1102.0,1056.0,1148.0,
|
| 48 |
-
13,LOCO,Search,Leave-one-out,1089.0,1043.0,1134.0,
|
| 49 |
-
14,LaplacianScore,Non-search,Distance,1042.0,976.0,1098.0,
|
| 50 |
-
15,Random,Baseline,Baseline,1000.0,941.0,1052.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1220.0'>1220<span class='sub'>+53 / −48</span></td><td class='num' data-sort='10.6'>10.6</td><td class='num shaded' data-sort='0.597' style='background:rgba(0,114,178,0.48)'>0.597</td><td class='num shaded' data-sort='0.507' style='background:rgba(213,94,0,0.42)'>0.507</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+48 / −44</span></td><td class='num' data-sort='8.0'>8.0</td><td class='num shaded' data-sort='0.562' style='background:rgba(0,114,178,0.46)'>0.562</td><td class='num shaded' data-sort='0.689' style='background:rgba(213,94,0,0.54)'>0.689</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1187.0'>1187<span class='sub'>+48 / −39</span></td><td class='num' data-sort='9.5'>9.5</td><td class='num shaded' data-sort='0.308' style='background:rgba(0,114,178,0.29)'>0.308</td><td class='num shaded' data-sort='0.568' style='background:rgba(213,94,0,0.46)'>0.568</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1182.0'>1182<span class='sub'>+43 / −42</span></td><td class='num' data-sort='7.5'>7.5</td><td class='num shaded' data-sort='0.587' style='background:rgba(0,114,178,0.47)'>0.587</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1176.0'>1176<span class='sub'>+46 / −47</span></td><td class='num' data-sort='9.5'>9.5</td><td class='num shaded' data-sort='0.651' style='background:rgba(0,114,178,0.52)'>0.651</td><td class='num shaded' data-sort='0.53' style='background:rgba(213,94,0,0.44)'>0.530</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='9.9'>9.9</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1170.0'>1170<span class='sub'>+54 / −46</span></td><td class='num' data-sort='6.8'>6.8</td><td class='num shaded' data-sort='0.246' style='background:rgba(0,114,178,0.24)'>0.246</td><td class='num shaded' data-sort='0.563' style='background:rgba(213,94,0,0.46)'>0.563</td><td class='num' data-sort='1.8'>1.8</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+49 / −45</span></td><td class='num' data-sort='6.9'>6.9</td><td class='num shaded' data-sort='0.324' style='background:rgba(0,114,178,0.30)'>0.324</td><td class='num shaded' data-sort='0.482' style='background:rgba(213,94,0,0.40)'>0.482</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1138.0'>1138<span class='sub'>+52 / −45</span></td><td class='num' data-sort='6.5'>6.5</td><td class='num shaded' data-sort='0.612' style='background:rgba(0,114,178,0.49)'>0.612</td><td class='num shaded' data-sort='0.594' style='background:rgba(213,94,0,0.48)'>0.594</td><td class='num' data-sort='13.4'>13.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1121.0'>1121<span class='sub'>+49 / −49</span></td><td class='num' data-sort='7.4'>7.4</td><td class='num shaded' data-sort='0.55' style='background:rgba(0,114,178,0.45)'>0.550</td><td class='num shaded' data-sort='0.659' style='background:rgba(213,94,0,0.52)'>0.659</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1114.0'>1114<span class='sub'>+49 / −53</span></td><td class='num' data-sort='7.4'>7.4</td><td class='num shaded' data-sort='0.404' style='background:rgba(0,114,178,0.35)'>0.404</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1111.0'>1111<span class='sub'>+45 / −53</span></td><td class='num' data-sort='3.3'>3.3</td><td class='num shaded' data-sort='0.059' style='background:rgba(0,114,178,0.12)'>0.059</td><td class='num shaded' data-sort='0.517' style='background:rgba(213,94,0,0.43)'>0.517</td><td class='num' data-sort='1035.1'>1,035.1</td><td class='num' data-sort='41.3'>41.3</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1102.0'>1102<span class='sub'>+46 / −46</span></td><td class='num' data-sort='4.0'>4.0</td><td class='num shaded' data-sort='0.176' style='background:rgba(0,114,178,0.20)'>0.176</td><td class='num shaded' data-sort='0.525' style='background:rgba(213,94,0,0.43)'>0.525</td><td class='num' data-sort='3042.8'>3,042.8</td><td class='num' data-sort='48.8'>48.8</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1089.0'>1089<span class='sub'>+45 / −46</span></td><td class='num' data-sort='3.6'>3.6</td><td class='num shaded' data-sort='0.08' style='background:rgba(0,114,178,0.13)'>0.080</td><td class='num shaded' data-sort='0.52' style='background:rgba(213,94,0,0.43)'>0.520</td><td class='num' data-sort='234.7'>234.7</td><td class='num' data-sort='18.3'>18.3</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1042.0'>1042<span class='sub'>+56 / −66</span></td><td class='num' data-sort='-39154926337779.6'>-39,154,926,337,779.6</td><td class='num shaded' data-sort='0.678' style='background:rgba(0,114,178,0.53)'>0.678</td><td class='num shaded' data-sort='0.701' style='background:rgba(213,94,0,0.55)'>0.701</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+52 / −59</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.002' style='background:rgba(0,114,178,0.08)'>0.002</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,RFImportance,Non-search,Tree,1220.0,1172.0,1273.0,10.6,0.597,0.507,0.2,0.0
|
| 37 |
+
2,F-Test,Non-search,Correlation,1189.0,1145.0,1237.0,8.0,0.562,0.689,0.0,0.0
|
| 38 |
+
3,ElasticNet,Non-search,Regularization,1187.0,1148.0,1235.0,9.5,0.308,0.568,6.2,0.0
|
| 39 |
+
4,MI,Non-search,Information theory,1182.0,1140.0,1225.0,7.5,0.587,0.627,0.4,0.0
|
| 40 |
+
5,mRMR,Non-search,Information theory,1176.0,1129.0,1222.0,9.5,0.651,0.53,0.4,9.9
|
| 41 |
+
6,Lasso,Non-search,Regularization,1170.0,1124.0,1224.0,6.8,0.246,0.563,1.8,0.0
|
| 42 |
+
7,CART,Non-search,Tree,1169.0,1124.0,1218.0,6.9,0.324,0.482,0.1,0.0
|
| 43 |
+
8,(R)ReliefF,Non-search,Distance,1138.0,1093.0,1190.0,6.5,0.612,0.594,13.4,0.0
|
| 44 |
+
9,GainRatio,Non-search,Information theory,1121.0,1072.0,1170.0,7.4,0.55,0.659,0.1,0.0
|
| 45 |
+
10,MarkovBlanket,Search,Backward search,1114.0,1061.0,1163.0,7.4,0.404,0.463,0.1,0.0
|
| 46 |
+
11,SBE,Search,Backward search,1111.0,1058.0,1156.0,3.3,0.059,0.517,1035.1,41.3
|
| 47 |
+
12,SFS,Search,Forward search,1102.0,1056.0,1148.0,4.0,0.176,0.525,3042.8,48.8
|
| 48 |
+
13,LOCO,Search,Leave-one-out,1089.0,1043.0,1134.0,3.6,0.08,0.52,234.7,18.3
|
| 49 |
+
14,LaplacianScore,Non-search,Distance,1042.0,976.0,1098.0,-39154926337779.6,0.678,0.701,0.2,0.0
|
| 50 |
+
15,Random,Baseline,Baseline,1000.0,941.0,1052.0,0.0,0.002,0.5,0.0,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_linearmodel/budgets_4/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='159.1' x2='159.1' y1='16.2' y2='368.0'/><text class='sa-text' x='159.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='258.9' x2='258.9' y1='16.2' y2='368.0'/><text class='sa-text' x='258.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='358.6' x2='358.6' y1='16.2' y2='368.0'/><text class='sa-text' x='358.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='458.3' x2='458.3' y1='16.2' y2='368.0'/><text class='sa-text' x='458.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='558.1' x2='558.1' y1='16.2' y2='368.0'/><text class='sa-text' x='558.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.8' x2='657.8' y1='16.2' y2='368.0'/><text class='sa-text' x='657.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.5' x2='757.5' y1='16.2' y2='368.0'/><text class='sa-text' x='757.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='291.5' y2='291.5'/><text class='sa-text' x='57.9' y='297.5' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='231.1' y2='231.1'/><text class='sa-text' x='57.9' y='237.1' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='170.6' y2='170.6'/><text class='sa-text' x='57.9' y='176.6' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='110.1' y2='110.1'/><text class='sa-text' x='57.9' y='116.1' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='49.6' y2='49.6'/><text class='sa-text' x='57.9' y='55.6' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 340.4,109.6 516.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='688.7' cy='64.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='563.6' cy='65.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='340.4' cy='109.6' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='524.0' cy='123.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.7' cy='127.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='483.4' cy='127.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.3' cy='143.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='687.2' cy='147.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.4' cy='154.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='231.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.5' cy='243.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='779.5' cy='251.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.8' y1='32.2' x2='394.0' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='394.0' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='688.7' y1='64.4' x2='767.4' y2='75.8' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='767.4' y='75.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='563.6' y1='65.1' x2='603.3' y2='50.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='603.3' y='50.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='340.4' y1='109.6' x2='380.1' y2='95.3' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='380.1' y='95.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='524.0' y1='123.8' x2='640.9' y2='119.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='640.9' y='119.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='468.7' y1='127.0' x2='514.0' y2='91.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='514.0' y='91.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><line class='sa-ink' x1='483.4' y1='127.0' x2='501.9' y2='139.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='501.9' y='139.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='474.3' y1='143.8' x2='502.0' y2='166.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='502.0' y='166.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='687.2' y1='147.5' x2='653.2' y2='161.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='653.2' y='161.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><text class='sa-dim sa-halo' x='796.9' y='169.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='231.0' x2='828.6' y2='198.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='198.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='737.5' y1='243.4' x2='765.8' y2='238.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='765.8' y='238.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='779.5' y1='251.6' x2='805.9' y2='268.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='805.9' y='268.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 10.6% Selection time per 1k features: 38.5 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 10.6% Selection time per 1k features: 38.5 s Datasets: 71'/><circle class='sa-hit' cx='688.7' cy='64.4' r='10.5' data-tip='ElasticNet Improvement over Random: 9.5% Selection time per 1k features: 2.04e+03 s Datasets: 71' aria-label='ElasticNet Improvement over Random: 9.5% Selection time per 1k features: 2.04e+03 s Datasets: 71'/><circle class='sa-hit' cx='563.6' cy='65.1' r='10.5' data-tip='mRMR Improvement over Random: 9.5% Selection time per 1k features: 114 s Datasets: 71' aria-label='mRMR Improvement over Random: 9.5% Selection time per 1k features: 114 s Datasets: 71'/><circle class='sa-hit' cx='340.4' cy='109.6' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 8.0% Selection time per 1k features: 0.657 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 8.0% Selection time per 1k features: 0.657 s Datasets: 71'/><circle class='sa-hit' cx='524.0' cy='123.8' r='10.5' data-tip='MI Improvement over Random: 7.5% Selection time per 1k features: 45.5 s Datasets: 71' aria-label='MI Improvement over Random: 7.5% Selection time per 1k features: 45.5 s Datasets: 71'/><circle class='sa-hit' cx='468.7' cy='127.0' r='10.5' data-tip='GainRatio Improvement over Random: 7.4% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 7.4%, 17.9 s Next to CART: 6.9%, 14.4 s' aria-label='GainRatio Improvement over Random: 7.4% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 7.4%, 17.9 s Next to CART: 6.9%, 14.4 s'/><circle class='sa-hit' cx='483.4' cy='127.0' r='10.5' data-tip='MarkovBlanket Improvement over Random: 7.4% Selection time per 1k features: 17.9 s Datasets: 71 Next to GainRatio: 7.4%, 12.7 s Next to CART: 6.9%, 14.4 s' aria-label='MarkovBlanket Improvement over Random: 7.4% Selection time per 1k features: 17.9 s Datasets: 71 Next to GainRatio: 7.4%, 12.7 s Next to CART: 6.9%, 14.4 s'/><circle class='sa-hit' cx='474.3' cy='143.8' r='10.5' data-tip='CART Improvement over Random: 6.9% Selection time per 1k features: 14.4 s Datasets: 71 Next to GainRatio: 7.4%, 12.7 s Next to MarkovBlanket: 7.4%, 17.9 s' aria-label='CART Improvement over Random: 6.9% Selection time per 1k features: 14.4 s Datasets: 71 Next to GainRatio: 7.4%, 12.7 s Next to MarkovBlanket: 7.4%, 17.9 s'/><circle class='sa-hit' cx='687.2' cy='147.5' r='10.5' data-tip='Lasso Improvement over Random: 6.8% Selection time per 1k features: 1.97e+03 s Datasets: 71' aria-label='Lasso Improvement over Random: 6.8% Selection time per 1k features: 1.97e+03 s Datasets: 71'/><circle class='sa-hit' cx='795.4' cy='154.8' r='10.5' data-tip='(R)ReliefF Improvement over Random: 6.5% Selection time per 1k features: 2.4e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 6.5% Selection time per 1k features: 2.4e+04 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='231.0' r='10.5' data-tip='SFS Improvement over Random: 4.0% Selection time per 1k features: 3.54e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 4.0% Selection time per 1k features: 3.54e+04 s Datasets: 71'/><circle class='sa-hit' cx='737.5' cy='243.4' r='10.5' data-tip='LOCO Improvement over Random: 3.6% Selection time per 1k features: 6.3e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 3.6% Selection time per 1k features: 6.3e+03 s Datasets: 71'/><circle class='sa-hit' cx='779.5' cy='251.6' r='10.5' data-tip='SBE Improvement over Random: 3.3% Selection time per 1k features: 1.66e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 3.3% Selection time per 1k features: 1.66e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,10.5744,38.5313,71,True
|
| 37 |
ElasticNet,9.5109,2038.5127,71,False
|
| 38 |
mRMR,9.488,113.6585,71,False
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='159.1' x2='159.1' y1='16.2' y2='368.0'/><text class='sa-text' x='159.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='258.9' x2='258.9' y1='16.2' y2='368.0'/><text class='sa-text' x='258.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='358.6' x2='358.6' y1='16.2' y2='368.0'/><text class='sa-text' x='358.6' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='458.3' x2='458.3' y1='16.2' y2='368.0'/><text class='sa-text' x='458.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='558.1' x2='558.1' y1='16.2' y2='368.0'/><text class='sa-text' x='558.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='657.8' x2='657.8' y1='16.2' y2='368.0'/><text class='sa-text' x='657.8' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='757.5' x2='757.5' y1='16.2' y2='368.0'/><text class='sa-text' x='757.5' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='291.5' y2='291.5'/><text class='sa-text' x='57.9' y='297.5' text-anchor='end' font-size='15'>2</text><line class='sa-grid' x1='66.9' x2='847.8' y1='231.1' y2='231.1'/><text class='sa-text' x='57.9' y='237.1' text-anchor='end' font-size='15'>4</text><line class='sa-grid' x1='66.9' x2='847.8' y1='170.6' y2='170.6'/><text class='sa-text' x='57.9' y='176.6' text-anchor='end' font-size='15'>6</text><line class='sa-grid' x1='66.9' x2='847.8' y1='110.1' y2='110.1'/><text class='sa-text' x='57.9' y='116.1' text-anchor='end' font-size='15'>8</text><line class='sa-grid' x1='66.9' x2='847.8' y1='49.6' y2='49.6'/><text class='sa-text' x='57.9' y='55.6' text-anchor='end' font-size='15'>10</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 340.4,109.6 516.8,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='516.8' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='688.7' cy='64.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='563.6' cy='65.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='340.4' cy='109.6' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='524.0' cy='123.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='468.7' cy='127.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='483.4' cy='127.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.3' cy='143.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='687.2' cy='147.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='795.4' cy='154.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='231.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='737.5' cy='243.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='779.5' cy='251.6' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='516.8' y1='32.2' x2='394.6' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='394.6' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='688.7' y1='64.4' x2='768.0' y2='76.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='768.0' y='76.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='563.6' y1='65.1' x2='603.3' y2='50.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='603.3' y='50.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='340.4' y1='109.6' x2='380.1' y2='95.3' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='380.1' y='95.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='524.0' y1='123.8' x2='644.7' y2='123.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='644.7' y='123.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='468.7' y1='127.0' x2='510.2' y2='88.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='510.2' y='88.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='501.9' y='131.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='474.3' y1='143.8' x2='499.4' y2='169.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='499.4' y='169.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='687.2' y1='147.5' x2='651.9' y2='161.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='651.9' y='161.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><text class='sa-dim sa-halo' x='796.9' y='168.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='812.3' y1='231.0' x2='828.6' y2='201.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='828.6' y='201.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='737.5' y1='243.4' x2='773.4' y2='236.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='773.4' y='236.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='779.5' y1='251.6' x2='805.9' y2='267.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='805.9' y='267.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='516.8' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 10.6% Selection time per 1k features: 38.5 s Datasets: 71' aria-label='RFImportance (on the Pareto front) Improvement over Random: 10.6% Selection time per 1k features: 38.5 s Datasets: 71'/><circle class='sa-hit' cx='688.7' cy='64.4' r='10.5' data-tip='ElasticNet Improvement over Random: 9.5% Selection time per 1k features: 2.04e+03 s Datasets: 71' aria-label='ElasticNet Improvement over Random: 9.5% Selection time per 1k features: 2.04e+03 s Datasets: 71'/><circle class='sa-hit' cx='563.6' cy='65.1' r='10.5' data-tip='mRMR Improvement over Random: 9.5% Selection time per 1k features: 114 s Datasets: 71' aria-label='mRMR Improvement over Random: 9.5% Selection time per 1k features: 114 s Datasets: 71'/><circle class='sa-hit' cx='340.4' cy='109.6' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 8.0% Selection time per 1k features: 0.657 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 8.0% Selection time per 1k features: 0.657 s Datasets: 71'/><circle class='sa-hit' cx='524.0' cy='123.8' r='10.5' data-tip='MI Improvement over Random: 7.5% Selection time per 1k features: 45.5 s Datasets: 71' aria-label='MI Improvement over Random: 7.5% Selection time per 1k features: 45.5 s Datasets: 71'/><circle class='sa-hit' cx='468.7' cy='127.0' r='10.5' data-tip='GainRatio Improvement over Random: 7.4% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 7.4%, 17.9 s Next to CART: 6.9%, 14.4 s' aria-label='GainRatio Improvement over Random: 7.4% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 7.4%, 17.9 s Next to CART: 6.9%, 14.4 s'/><circle class='sa-hit' cx='483.4' cy='127.0' r='10.5' data-tip='MarkovBlanket Improvement over Random: 7.4% Selection time per 1k features: 17.9 s Datasets: 71 Next to GainRatio: 7.4%, 12.7 s Next to CART: 6.9%, 14.4 s' aria-label='MarkovBlanket Improvement over Random: 7.4% Selection time per 1k features: 17.9 s Datasets: 71 Next to GainRatio: 7.4%, 12.7 s Next to CART: 6.9%, 14.4 s'/><circle class='sa-hit' cx='474.3' cy='143.8' r='10.5' data-tip='CART Improvement over Random: 6.9% Selection time per 1k features: 14.4 s Datasets: 71 Next to GainRatio: 7.4%, 12.7 s Next to MarkovBlanket: 7.4%, 17.9 s' aria-label='CART Improvement over Random: 6.9% Selection time per 1k features: 14.4 s Datasets: 71 Next to GainRatio: 7.4%, 12.7 s Next to MarkovBlanket: 7.4%, 17.9 s'/><circle class='sa-hit' cx='687.2' cy='147.5' r='10.5' data-tip='Lasso Improvement over Random: 6.8% Selection time per 1k features: 1.97e+03 s Datasets: 71' aria-label='Lasso Improvement over Random: 6.8% Selection time per 1k features: 1.97e+03 s Datasets: 71'/><circle class='sa-hit' cx='795.4' cy='154.8' r='10.5' data-tip='(R)ReliefF Improvement over Random: 6.5% Selection time per 1k features: 2.4e+04 s Datasets: 71' aria-label='(R)ReliefF Improvement over Random: 6.5% Selection time per 1k features: 2.4e+04 s Datasets: 71'/><circle class='sa-hit' cx='812.3' cy='231.0' r='10.5' data-tip='SFS Improvement over Random: 4.0% Selection time per 1k features: 3.54e+04 s Datasets: 71' aria-label='SFS Improvement over Random: 4.0% Selection time per 1k features: 3.54e+04 s Datasets: 71'/><circle class='sa-hit' cx='737.5' cy='243.4' r='10.5' data-tip='LOCO Improvement over Random: 3.6% Selection time per 1k features: 6.3e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 3.6% Selection time per 1k features: 6.3e+03 s Datasets: 71'/><circle class='sa-hit' cx='779.5' cy='251.6' r='10.5' data-tip='SBE Improvement over Random: 3.3% Selection time per 1k features: 1.66e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 3.3% Selection time per 1k features: 1.66e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0027 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,10.5744,38.5313,71,True
|
| 37 |
ElasticNet,9.5109,2038.5127,71,False
|
| 38 |
mRMR,9.488,113.6585,71,False
|
data/subsets/tasks_all/models_linearmodel/budgets_4/website_leaderboard.csv
CHANGED
|
@@ -1,16 +1,16 @@
|
|
| 1 |
-
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
-
1,RFImportance,Non-search,Tree,1220.0,1172.0,1273.0,6.72,59.2,10.2,,0.597,0.507,0.2,0.0,71,213
|
| 3 |
-
2,F-Test,Non-search,Correlation,1189.0,1145.0,1237.0,7.21,55.6,11.1,,0.562,0.689,0.0,0.0,71,213
|
| 4 |
-
3,ElasticNet,Non-search,Regularization,1187.0,1148.0,1235.0,7.06,56.7,10.5,,0.308,0.568,6.2,0.0,71,213
|
| 5 |
-
4,MI,Non-search,Information theory,1182.0,1140.0,1225.0,7.14,56.1,11.0,,0.587,0.627,0.4,0.0,71,213
|
| 6 |
-
5,mRMR,Non-search,Information theory,1176.0,1129.0,1222.0,7.62,52.8,10.6,,0.651,0.53,0.4,9.9,71,213
|
| 7 |
-
6,Lasso,Non-search,Regularization,1170.0,1124.0,1224.0,7.1,56.4,11.6,,0.246,0.563,1.8,0.0,71,213
|
| 8 |
-
7,CART,Non-search,Tree,1169.0,1124.0,1218.0,7.55,53.2,12.6,,0.324,0.482,0.1,0.0,71,213
|
| 9 |
-
8,(R)ReliefF,Non-search,Distance,1138.0,1093.0,1190.0,8.11,49.2,11.9,,0.612,0.594,13.4,0.0,71,213
|
| 10 |
-
9,GainRatio,Non-search,Information theory,1121.0,1072.0,1170.0,8.42,47.0,12.9,,0.55,0.659,0.1,0.0,71,213
|
| 11 |
-
10,MarkovBlanket,Search,Backward search,1114.0,1061.0,1163.0,8.54,46.2,12.6,,0.404,0.463,0.1,0.0,71,213
|
| 12 |
-
11,SBE,Search,Backward search,1111.0,1058.0,1156.0,8.55,46.1,14.8,,0.059,0.517,1035.1,41.3,71,213
|
| 13 |
-
12,SFS,Search,Forward search,1102.0,1056.0,1148.0,8.47,46.7,15.2,,0.176,0.525,3042.8,48.8,71,213
|
| 14 |
-
13,LOCO,Search,Leave-one-out,1089.0,1043.0,1134.0,8.29,47.9,15.2,,0.08,0.52,234.7,18.3,71,213
|
| 15 |
-
14,LaplacianScore,Non-search,Distance,1042.0,976.0,1098.0,9.77,37.4,20.5,,0.678,0.701,0.2,0.0,71,213
|
| 16 |
-
15,Random,Baseline,Baseline,1000.0,941.0,1052.0,9.46,39.5,17.7,,0.002,0.5,0.0,0.0,71,213
|
|
|
|
| 1 |
+
Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Improvement over Random (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 2 |
+
1,RFImportance,Non-search,Tree,1220.0,1172.0,1273.0,6.72,59.2,10.2,10.6,,0.597,0.507,0.2,0.0,71,213
|
| 3 |
+
2,F-Test,Non-search,Correlation,1189.0,1145.0,1237.0,7.21,55.6,11.1,8.0,,0.562,0.689,0.0,0.0,71,213
|
| 4 |
+
3,ElasticNet,Non-search,Regularization,1187.0,1148.0,1235.0,7.06,56.7,10.5,9.5,,0.308,0.568,6.2,0.0,71,213
|
| 5 |
+
4,MI,Non-search,Information theory,1182.0,1140.0,1225.0,7.14,56.1,11.0,7.5,,0.587,0.627,0.4,0.0,71,213
|
| 6 |
+
5,mRMR,Non-search,Information theory,1176.0,1129.0,1222.0,7.62,52.8,10.6,9.5,,0.651,0.53,0.4,9.9,71,213
|
| 7 |
+
6,Lasso,Non-search,Regularization,1170.0,1124.0,1224.0,7.1,56.4,11.6,6.8,,0.246,0.563,1.8,0.0,71,213
|
| 8 |
+
7,CART,Non-search,Tree,1169.0,1124.0,1218.0,7.55,53.2,12.6,6.9,,0.324,0.482,0.1,0.0,71,213
|
| 9 |
+
8,(R)ReliefF,Non-search,Distance,1138.0,1093.0,1190.0,8.11,49.2,11.9,6.5,,0.612,0.594,13.4,0.0,71,213
|
| 10 |
+
9,GainRatio,Non-search,Information theory,1121.0,1072.0,1170.0,8.42,47.0,12.9,7.4,,0.55,0.659,0.1,0.0,71,213
|
| 11 |
+
10,MarkovBlanket,Search,Backward search,1114.0,1061.0,1163.0,8.54,46.2,12.6,7.4,,0.404,0.463,0.1,0.0,71,213
|
| 12 |
+
11,SBE,Search,Backward search,1111.0,1058.0,1156.0,8.55,46.1,14.8,3.3,,0.059,0.517,1035.1,41.3,71,213
|
| 13 |
+
12,SFS,Search,Forward search,1102.0,1056.0,1148.0,8.47,46.7,15.2,4.0,,0.176,0.525,3042.8,48.8,71,213
|
| 14 |
+
13,LOCO,Search,Leave-one-out,1089.0,1043.0,1134.0,8.29,47.9,15.2,3.6,,0.08,0.52,234.7,18.3,71,213
|
| 15 |
+
14,LaplacianScore,Non-search,Distance,1042.0,976.0,1098.0,9.77,37.4,20.5,-39154926337779.6,,0.678,0.701,0.2,0.0,71,213
|
| 16 |
+
15,Random,Baseline,Baseline,1000.0,941.0,1052.0,9.46,39.5,17.7,0.0,,0.002,0.5,0.0,0.0,71,213
|
data/subsets/tasks_all/models_linearmodel/budgets_all/leaderboard_table.html
CHANGED
|
@@ -32,22 +32,22 @@ td.na { color:var(--muted); }
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
-
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean rank of the downstream error.'>Avg. rank ↓</th><th class='num' title='Mean relative gap (error − best error) / error to the best method, in %.'>Improvability ↓</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1286.0'>1286<span class='sub'>+43 / −39</span></td><td class='num' data-sort='6.26'>6.26</td><td class='num' data-sort='14.7'>14.7</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1282.0'>1282<span class='sub'>+44 / −36</span></td><td class='num' data-sort='6.53'>6.53</td><td class='num' data-sort='14.2'>14.2</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1259.0'>1259<span class='sub'>+43 / −41</span></td><td class='num' data-sort='6.57'>6.57</td><td class='num' data-sort='15.4'>15.4</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1257.0'>1257<span class='sub'>+32 / −30</span></td><td class='num' data-sort='6.8'>6.80</td><td class='num' data-sort='16.1'>16.1</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1246.0'>1246<span class='sub'>+32 / −32</span></td><td class='num' data-sort='7.02'>7.02</td><td class='num' data-sort='16.7'>16.7</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1232.0'>1232<span class='sub'>+41 / −36</span></td><td class='num' data-sort='7.2'>7.20</td><td class='num' data-sort='17.6'>17.6</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1213.0'>1213<span class='sub'>+34 / −34</span></td><td class='num' data-sort='7.6'>7.60</td><td class='num' data-sort='17.3'>17.3</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1196.0'>1196<span class='sub'>+33 / −32</span></td><td class='num' data-sort='7.94'>7.94</td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+33 / −31</span></td><td class='num' data-sort='8.14'>8.14</td><td class='num' data-sort='19.8'>19.8</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+35 / −36</span></td><td class='num' data-sort='7.9'>7.90</td><td class='num' data-sort='19.7'>19.7</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2071.2'>2,071.2</td><td class='num' data-sort='43.6'>43.6</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+33 / −35</span></td><td class='num' data-sort='8.51'>8.51</td><td class='num' data-sort='19.2'>19.2</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1135.0'>1135<span class='sub'>+33 / −36</span></td><td class='num' data-sort='8.63'>8.63</td><td class='num' data-sort='23.0'>23.0</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='7.2'>7.2</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1130.0'>1130<span class='sub'>+34 / −40</span></td><td class='num' data-sort='8.89'>8.89</td><td class='num' data-sort='23.7'>23.7</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2561.3'>2,561.3</td><td class='num' data-sort='46.7'>46.7</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+36 / −39</span></td><td class='num' data-sort='10.67'>10.67</td><td class='num' data-sort='30.1'>30.1</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='994.0'>994<span class='sub'>+46 / −54</span></td><td class='num' data-sort='11.34'>11.34</td><td class='num' data-sort='34.1'>34.1</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons
|
| 36 |
-
1,ElasticNet,Non-search,Regularization,1286.0,1247.0,1329.0,
|
| 37 |
-
2,RFImportance,Non-search,Tree,1282.0,1246.0,1326.0,
|
| 38 |
-
3,Lasso,Non-search,Regularization,1259.0,1218.0,1302.0,
|
| 39 |
-
4,MI,Non-search,Information theory,1257.0,1227.0,1289.0,
|
| 40 |
-
5,F-Test,Non-search,Correlation,1246.0,1214.0,1278.0,
|
| 41 |
-
6,CART,Non-search,Tree,1232.0,1196.0,1273.0,
|
| 42 |
-
7,mRMR,Non-search,Information theory,1213.0,1179.0,1247.0,
|
| 43 |
-
8,MarkovBlanket,Search,Backward search,1196.0,1164.0,1229.0,
|
| 44 |
-
9,GainRatio,Non-search,Information theory,1189.0,1158.0,1222.0,
|
| 45 |
-
10,SFS,Search,Forward search,1186.0,1150.0,1221.0,
|
| 46 |
-
11,(R)ReliefF,Non-search,Distance,1169.0,1134.0,1202.0,
|
| 47 |
-
12,LOCO,Search,Leave-one-out,1135.0,1099.0,1168.0,
|
| 48 |
-
13,SBE,Search,Backward search,1130.0,1090.0,1164.0,8.
|
| 49 |
-
14,Random,Baseline,Baseline,1000.0,961.0,1036.0,
|
| 50 |
-
15,LaplacianScore,Non-search,Distance,994.0,940.0,1040.0,
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
|
|
|
| 32 |
.pill.cat-baseline { color:var(--muted); }
|
| 33 |
.medal { display:inline-block; width:1.4em; text-align:center; font-size:15px; line-height:1; }
|
| 34 |
td.shaded { font-weight:600; }
|
| 35 |
+
</style></head><body><div class='toolbar'><span class='hint'>Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.</span><button onclick="saCsv('selectarena_leaderboard.csv')">CSV</button></div><div class='wrap'><table id='sa-table'><thead><tr><th class='num' title='Rank by Elo.'>#</th><th class=''>Method</th><th class='' title='Taxonomy of the paper (Appendix A). Search methods evaluate many feature subsets with the model (forward, backward, leave-one-out); non-search methods use a proxy instead (correlation, distance, information theory, tree, regularization).'>Type</th><th class='num' title='Bradley-Terry Elo from pairwise matches within each dataset, budget and downstream model. Random = 1000. +/− give the 95% bootstrap interval.'>Elo ↑</th><th class='num' title='Mean reduction of the downstream error compared with random selection on the same dataset, in % (as in the Pareto figure; Random is 0).'>Improvement over Random ↑</th><th class='num' title='Agreement of the selected features across bootstrap resamples (Nogueira et al., 2018).'>Stability ↑</th><th class='num' title='Share of the budget filled with original features when as many noise features are added.'>Validity ↑</th><th class='num' title='Median wall-clock time of the selection.'>Time (s) ↓</th><th class='num' title='Share of runs where the method ran out of time and random features filled the budget.'>Fallback (%)</th></tr></thead><tbody><tr><td class='num' data-sort='1'><span class='medal' role='img' aria-label='gold medal'>🥇</span>1</td><td class='method' data-sort='ElasticNet'>ElasticNet</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1286.0'>1286<span class='sub'>+43 / −39</span></td><td class='num' data-sort='22.6'>22.6</td><td class='num shaded' data-sort='0.447' style='background:rgba(0,114,178,0.38)'>0.447</td><td class='num shaded' data-sort='0.76' style='background:rgba(213,94,0,0.59)'>0.760</td><td class='num' data-sort='6.2'>6.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='2'><span class='medal' role='img' aria-label='silver medal'>🥈</span>2</td><td class='method' data-sort='RFImportance'>RFImportance</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1282.0'>1282<span class='sub'>+44 / −36</span></td><td class='num' data-sort='24.6'>24.6</td><td class='num shaded' data-sort='0.643' style='background:rgba(0,114,178,0.51)'>0.643</td><td class='num shaded' data-sort='0.669' style='background:rgba(213,94,0,0.53)'>0.669</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='3'><span class='medal' role='img' aria-label='bronze medal'>🥉</span>3</td><td class='method' data-sort='Lasso'>Lasso</td><td class='' data-sort='Regularization'><span class='pill cat-non-search'>Regularization</span><span class='sub'>non-search</span></td><td class='num' data-sort='1259.0'>1259<span class='sub'>+43 / −41</span></td><td class='num' data-sort='21.7'>21.7</td><td class='num shaded' data-sort='0.362' style='background:rgba(0,114,178,0.32)'>0.362</td><td class='num shaded' data-sort='0.73' style='background:rgba(213,94,0,0.57)'>0.730</td><td class='num' data-sort='1.9'>1.9</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='4'><span class='medal'></span>4</td><td class='method' data-sort='MI'>MI</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1257.0'>1257<span class='sub'>+32 / −30</span></td><td class='num' data-sort='23.2'>23.2</td><td class='num shaded' data-sort='0.647' style='background:rgba(0,114,178,0.51)'>0.647</td><td class='num shaded' data-sort='0.785' style='background:rgba(213,94,0,0.61)'>0.785</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='5'><span class='medal'></span>5</td><td class='method' data-sort='F-Test'>F-Test</td><td class='' data-sort='Correlation'><span class='pill cat-non-search'>Correlation</span><span class='sub'>non-search</span></td><td class='num' data-sort='1246.0'>1246<span class='sub'>+32 / −32</span></td><td class='num' data-sort='22.9'>22.9</td><td class='num shaded' data-sort='0.679' style='background:rgba(0,114,178,0.53)'>0.679</td><td class='num shaded' data-sort='0.846' style='background:rgba(213,94,0,0.65)'>0.846</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='6'><span class='medal'></span>6</td><td class='method' data-sort='CART'>CART</td><td class='' data-sort='Tree'><span class='pill cat-non-search'>Tree</span><span class='sub'>non-search</span></td><td class='num' data-sort='1232.0'>1232<span class='sub'>+41 / −36</span></td><td class='num' data-sort='22.0'>22.0</td><td class='num shaded' data-sort='0.422' style='background:rgba(0,114,178,0.36)'>0.422</td><td class='num shaded' data-sort='0.586' style='background:rgba(213,94,0,0.47)'>0.586</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='7'><span class='medal'></span>7</td><td class='method' data-sort='mRMR'>mRMR</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1213.0'>1213<span class='sub'>+34 / −34</span></td><td class='num' data-sort='21.8'>21.8</td><td class='num shaded' data-sort='0.604' style='background:rgba(0,114,178,0.48)'>0.604</td><td class='num shaded' data-sort='0.655' style='background:rgba(213,94,0,0.52)'>0.655</td><td class='num' data-sort='0.4'>0.4</td><td class='num' data-sort='3.4'>3.4</td></tr><tr><td class='num' data-sort='8'><span class='medal'></span>8</td><td class='method' data-sort='MarkovBlanket'>MarkovBlanket</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1196.0'>1196<span class='sub'>+33 / −32</span></td><td class='num' data-sort='19.0'>19.0</td><td class='num shaded' data-sort='0.413' style='background:rgba(0,114,178,0.36)'>0.413</td><td class='num shaded' data-sort='0.463' style='background:rgba(213,94,0,0.39)'>0.463</td><td class='num' data-sort='0.1'>0.1</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='9'><span class='medal'></span>9</td><td class='method' data-sort='GainRatio'>GainRatio</td><td class='' data-sort='Information theory'><span class='pill cat-non-search'>Information theory</span><span class='sub'>non-search</span></td><td class='num' data-sort='1189.0'>1189<span class='sub'>+33 / −31</span></td><td class='num' data-sort='18.4'>18.4</td><td class='num shaded' data-sort='0.644' style='background:rgba(0,114,178,0.51)'>0.644</td><td class='num shaded' data-sort='0.825' style='background:rgba(213,94,0,0.63)'>0.825</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='10'><span class='medal'></span>10</td><td class='method' data-sort='SFS'>SFS</td><td class='' data-sort='Forward search'><span class='pill cat-search'>Forward search</span><span class='sub'>search</span></td><td class='num' data-sort='1186.0'>1186<span class='sub'>+35 / −36</span></td><td class='num' data-sort='17.7'>17.7</td><td class='num shaded' data-sort='0.267' style='background:rgba(0,114,178,0.26)'>0.267</td><td class='num shaded' data-sort='0.627' style='background:rgba(213,94,0,0.50)'>0.627</td><td class='num' data-sort='2071.2'>2,071.2</td><td class='num' data-sort='43.6'>43.6</td></tr><tr><td class='num' data-sort='11'><span class='medal'></span>11</td><td class='method' data-sort='(R)ReliefF'>(R)ReliefF</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='1169.0'>1169<span class='sub'>+33 / −35</span></td><td class='num' data-sort='16.3'>16.3</td><td class='num shaded' data-sort='0.646' style='background:rgba(0,114,178,0.51)'>0.646</td><td class='num shaded' data-sort='0.74' style='background:rgba(213,94,0,0.58)'>0.740</td><td class='num' data-sort='13.2'>13.2</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='12'><span class='medal'></span>12</td><td class='method' data-sort='LOCO'>LOCO</td><td class='' data-sort='Leave-one-out'><span class='pill cat-search'>Leave-one-out</span><span class='sub'>search</span></td><td class='num' data-sort='1135.0'>1135<span class='sub'>+33 / −36</span></td><td class='num' data-sort='12.0'>12.0</td><td class='num shaded' data-sort='0.173' style='background:rgba(0,114,178,0.20)'>0.173</td><td class='num shaded' data-sort='0.583' style='background:rgba(213,94,0,0.47)'>0.583</td><td class='num' data-sort='242.0'>242.0</td><td class='num' data-sort='7.2'>7.2</td></tr><tr><td class='num' data-sort='13'><span class='medal'></span>13</td><td class='method' data-sort='SBE'>SBE</td><td class='' data-sort='Backward search'><span class='pill cat-search'>Backward search</span><span class='sub'>search</span></td><td class='num' data-sort='1130.0'>1130<span class='sub'>+34 / −40</span></td><td class='num' data-sort='8.4'>8.4</td><td class='num shaded' data-sort='0.132' style='background:rgba(0,114,178,0.17)'>0.132</td><td class='num shaded' data-sort='0.56' style='background:rgba(213,94,0,0.46)'>0.560</td><td class='num' data-sort='2561.3'>2,561.3</td><td class='num' data-sort='46.7'>46.7</td></tr><tr><td class='num' data-sort='14'><span class='medal'></span>14</td><td class='method' data-sort='Random'>Random</td><td class='' data-sort='Baseline'><span class='pill cat-baseline'>Baseline</span></td><td class='num' data-sort='1000.0'>1000<span class='sub'>+36 / −39</span></td><td class='num' data-sort='0.0'>0.0</td><td class='num shaded' data-sort='0.001' style='background:rgba(0,114,178,0.08)'>0.001</td><td class='num shaded' data-sort='0.5' style='background:rgba(213,94,0,0.42)'>0.500</td><td class='num' data-sort='0.0'>0.0</td><td class='num' data-sort='0.0'>0.0</td></tr><tr><td class='num' data-sort='15'><span class='medal'></span>15</td><td class='method' data-sort='LaplacianScore'>LaplacianScore</td><td class='' data-sort='Distance'><span class='pill cat-non-search'>Distance</span><span class='sub'>non-search</span></td><td class='num' data-sort='994.0'>994<span class='sub'>+46 / −54</span></td><td class='num' data-sort='-14.4'>-14.4</td><td class='num shaded' data-sort='0.652' style='background:rgba(0,114,178,0.52)'>0.652</td><td class='num shaded' data-sort='0.797' style='background:rgba(213,94,0,0.61)'>0.797</td><td class='num' data-sort='0.2'>0.2</td><td class='num' data-sort='0.0'>0.0</td></tr></tbody></table></div><script type='text/plain' id='sa-csv'>Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Improvement over Random (%),Stability,Validity,Selection time (s),Fallback rate (%)
|
| 36 |
+
1,ElasticNet,Non-search,Regularization,1286.0,1247.0,1329.0,22.6,0.447,0.76,6.2,0.0
|
| 37 |
+
2,RFImportance,Non-search,Tree,1282.0,1246.0,1326.0,24.6,0.643,0.669,0.2,0.0
|
| 38 |
+
3,Lasso,Non-search,Regularization,1259.0,1218.0,1302.0,21.7,0.362,0.73,1.9,0.0
|
| 39 |
+
4,MI,Non-search,Information theory,1257.0,1227.0,1289.0,23.2,0.647,0.785,0.4,0.0
|
| 40 |
+
5,F-Test,Non-search,Correlation,1246.0,1214.0,1278.0,22.9,0.679,0.846,0.0,0.0
|
| 41 |
+
6,CART,Non-search,Tree,1232.0,1196.0,1273.0,22.0,0.422,0.586,0.1,0.0
|
| 42 |
+
7,mRMR,Non-search,Information theory,1213.0,1179.0,1247.0,21.8,0.604,0.655,0.4,3.4
|
| 43 |
+
8,MarkovBlanket,Search,Backward search,1196.0,1164.0,1229.0,19.0,0.413,0.463,0.1,0.0
|
| 44 |
+
9,GainRatio,Non-search,Information theory,1189.0,1158.0,1222.0,18.4,0.644,0.825,0.2,0.0
|
| 45 |
+
10,SFS,Search,Forward search,1186.0,1150.0,1221.0,17.7,0.267,0.627,2071.2,43.6
|
| 46 |
+
11,(R)ReliefF,Non-search,Distance,1169.0,1134.0,1202.0,16.3,0.646,0.74,13.2,0.0
|
| 47 |
+
12,LOCO,Search,Leave-one-out,1135.0,1099.0,1168.0,12.0,0.173,0.583,242.0,7.2
|
| 48 |
+
13,SBE,Search,Backward search,1130.0,1090.0,1164.0,8.4,0.132,0.56,2561.3,46.7
|
| 49 |
+
14,Random,Baseline,Baseline,1000.0,961.0,1036.0,0.0,0.001,0.5,0.0,0.0
|
| 50 |
+
15,LaplacianScore,Non-search,Distance,994.0,940.0,1040.0,-14.4,0.652,0.797,0.2,0.0
|
| 51 |
</script><script>
|
| 52 |
// Theme: the Space sets data-theme on this page when its theme button was used (same origin), or
|
| 53 |
// sends {saTheme: 'light' | 'dark' | null}; without either, the page follows the system setting.
|
data/subsets/tasks_all/models_linearmodel/budgets_all/pareto.html
CHANGED
|
@@ -32,7 +32,7 @@ svg { display:block; width:100%; height:auto; }
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
-
</style><line class='sa-grid' x1='161.4' x2='161.4' y1='16.2' y2='368.0'/><text class='sa-text' x='161.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='262.3' x2='262.3' y1='16.2' y2='368.0'/><text class='sa-text' x='262.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='363.2' x2='363.2' y1='16.2' y2='368.0'/><text class='sa-text' x='363.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='464.1' x2='464.1' y1='16.2' y2='368.0'/><text class='sa-text' x='464.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='565.0' x2='565.0' y1='16.2' y2='368.0'/><text class='sa-text' x='565.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='665.9' x2='665.9' y1='16.2' y2='368.0'/><text class='sa-text' x='665.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='766.7' x2='766.7' y1='16.2' y2='368.0'/><text class='sa-text' x='766.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='286.9' y2='286.9'/><text class='sa-text' x='57.9' y='292.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='221.8' y2='221.8'/><text class='sa-text' x='57.9' y='227.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='156.7' y2='156.7'/><text class='sa-text' x='57.9' y='162.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='91.6' y2='91.6'/><text class='sa-text' x='57.9' y='97.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='26.5' y2='26.5'/><text class='sa-text' x='57.9' y='32.5' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 344.2,54.5 523.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='523.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='530.9' cy='50.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='344.2' cy='54.5' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='698.4' cy='58.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='480.3' cy='66.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='534.6' cy='68.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='696.5' cy='69.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='491.8' cy='104.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.6' cy='112.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='121.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='804.9' cy='140.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='746.8' cy='196.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='811.4' cy='242.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='523.5' y1='32.2' x2='636.9' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='636.9' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='530.9' y1='50.5' x2='644.3' y2='57.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='644.3' y='57.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='344.2' y1='54.5' x2='293.1' y2='59.1' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='293.1' y='59.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='698.4' y1='58.4' x2='778.2' y2='54.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='778.2' y='54.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><line class='sa-ink' x1='480.3' y1='66.1' x2='457.7' y2='51.9' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='457.7' y='51.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='534.6' y1='68.0' x2='564.8' y2='86.3' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='564.8' y='86.3' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='696.5' y1='69.7' x2='735.3' y2='84.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='735.3' y='84.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='491.8' y1='104.4' x2='377.2' y2='94.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='377.2' y='94.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='474.6' y1='112.9' x2='545.1' y2='126.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='545.1' y='126.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='825.5' y='107.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='804.9' y1='140.3' x2='740.9' y2='154.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='740.9' y='154.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='746.8' y1='196.3' x2='712.8' y2='210.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='712.8' y='210.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='811.4' y1='242.8' x2='786.2' y2='257.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='786.2' y='257.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='523.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 24.6% Selection time per 1k features: 38.8 s Datasets: 71 Next to MI: 23.2%, 46 s' aria-label='RFImportance (on the Pareto front) Improvement over Random: 24.6% Selection time per 1k features: 38.8 s Datasets: 71 Next to MI: 23.2%, 46 s'/><circle class='sa-hit' cx='530.9' cy='50.5' r='10.5' data-tip='MI Improvement over Random: 23.2% Selection time per 1k features: 46 s Datasets: 71 Next to RFImportance: 24.6%, 38.8 s Next to mRMR: 21.8%, 50 s' aria-label='MI Improvement over Random: 23.2% Selection time per 1k features: 46 s Datasets: 71 Next to RFImportance: 24.6%, 38.8 s Next to mRMR: 21.8%, 50 s'/><circle class='sa-hit' cx='344.2' cy='54.5' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 22.9% Selection time per 1k features: 0.647 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 22.9% Selection time per 1k features: 0.647 s Datasets: 71'/><circle class='sa-hit' cx='698.4' cy='58.4' r='10.5' data-tip='ElasticNet Improvement over Random: 22.6% Selection time per 1k features: 2.1e+03 s Datasets: 71 Next to Lasso: 21.7%, 2.01e+03 s' aria-label='ElasticNet Improvement over Random: 22.6% Selection time per 1k features: 2.1e+03 s Datasets: 71 Next to Lasso: 21.7%, 2.01e+03 s'/><circle class='sa-hit' cx='480.3' cy='66.1' r='10.5' data-tip='CART Improvement over Random: 22.0% Selection time per 1k features: 14.5 s Datasets: 71' aria-label='CART Improvement over Random: 22.0% Selection time per 1k features: 14.5 s Datasets: 71'/><circle class='sa-hit' cx='534.6' cy='68.0' r='10.5' data-tip='mRMR Improvement over Random: 21.8% Selection time per 1k features: 50 s Datasets: 71 Next to MI: 23.2%, 46 s' aria-label='mRMR Improvement over Random: 21.8% Selection time per 1k features: 50 s Datasets: 71 Next to MI: 23.2%, 46 s'/><circle class='sa-hit' cx='696.5' cy='69.7' r='10.5' data-tip='Lasso Improvement over Random: 21.7% Selection time per 1k features: 2.01e+03 s Datasets: 71 Next to ElasticNet: 22.6%, 2.1e+03 s' aria-label='Lasso Improvement over Random: 21.7% Selection time per 1k features: 2.01e+03 s Datasets: 71 Next to ElasticNet: 22.6%, 2.1e+03 s'/><circle class='sa-hit' cx='491.8' cy='104.4' r='10.5' data-tip='MarkovBlanket Improvement over Random: 19.0% Selection time per 1k features: 18.8 s Datasets: 71 Next to GainRatio: 18.4%, 12.7 s' aria-label='MarkovBlanket Improvement over Random: 19.0% Selection time per 1k features: 18.8 s Datasets: 71 Next to GainRatio: 18.4%, 12.7 s'/><circle class='sa-hit' cx='474.6' cy='112.9' r='10.5' data-tip='GainRatio Improvement over Random: 18.4% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 19.0%, 18.8 s' aria-label='GainRatio Improvement over Random: 18.4% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 19.0%, 18.8 s'/><circle class='sa-hit' cx='812.3' cy='121.4' r='10.5' data-tip='SFS Improvement over Random: 17.7% Selection time per 1k features: 2.83e+04 s Datasets: 71 Next to (R)ReliefF: 16.3%, 2.39e+04 s' aria-label='SFS Improvement over Random: 17.7% Selection time per 1k features: 2.83e+04 s Datasets: 71 Next to (R)ReliefF: 16.3%, 2.39e+04 s'/><circle class='sa-hit' cx='804.9' cy='140.3' r='10.5' data-tip='(R)ReliefF Improvement over Random: 16.3% Selection time per 1k features: 2.39e+04 s Datasets: 71 Next to SFS: 17.7%, 2.83e+04 s' aria-label='(R)ReliefF Improvement over Random: 16.3% Selection time per 1k features: 2.39e+04 s Datasets: 71 Next to SFS: 17.7%, 2.83e+04 s'/><circle class='sa-hit' cx='746.8' cy='196.3' r='10.5' data-tip='LOCO Improvement over Random: 12.0% Selection time per 1k features: 6.34e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 12.0% Selection time per 1k features: 6.34e+03 s Datasets: 71'/><circle class='sa-hit' cx='811.4' cy='242.8' r='10.5' data-tip='SBE Improvement over Random: 8.4% Selection time per 1k features: 2.77e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 8.4% Selection time per 1k features: 2.77e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,24.5664,38.8189,71,True
|
| 37 |
MI,23.1632,45.9696,71,False
|
| 38 |
F-Test,22.8523,0.6474,71,True
|
|
|
|
| 32 |
.sa-legend { fill:var(--bg); stroke:var(--line); }
|
| 33 |
.sa-heat { image-rendering:pixelated; image-rendering:crisp-edges; }
|
| 34 |
.sa-halo { paint-order:stroke; stroke:var(--bg); stroke-width:3px; stroke-opacity:0.7; }
|
| 35 |
+
</style><line class='sa-grid' x1='161.4' x2='161.4' y1='16.2' y2='368.0'/><text class='sa-text' x='161.4' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-2</tspan></text><line class='sa-grid' x1='262.3' x2='262.3' y1='16.2' y2='368.0'/><text class='sa-text' x='262.3' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>-1</tspan></text><line class='sa-grid' x1='363.2' x2='363.2' y1='16.2' y2='368.0'/><text class='sa-text' x='363.2' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>0</tspan></text><line class='sa-grid' x1='464.1' x2='464.1' y1='16.2' y2='368.0'/><text class='sa-text' x='464.1' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>1</tspan></text><line class='sa-grid' x1='565.0' x2='565.0' y1='16.2' y2='368.0'/><text class='sa-text' x='565.0' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>2</tspan></text><line class='sa-grid' x1='665.9' x2='665.9' y1='16.2' y2='368.0'/><text class='sa-text' x='665.9' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>3</tspan></text><line class='sa-grid' x1='766.7' x2='766.7' y1='16.2' y2='368.0'/><text class='sa-text' x='766.7' y='390.5' text-anchor='middle' font-size='15'>10<tspan dy='-7' font-size='10'>4</tspan></text><line class='sa-grid' x1='66.9' x2='847.8' y1='352.0' y2='352.0'/><text class='sa-text' x='57.9' y='358.0' text-anchor='end' font-size='15'>0</text><line class='sa-grid' x1='66.9' x2='847.8' y1='286.9' y2='286.9'/><text class='sa-text' x='57.9' y='292.9' text-anchor='end' font-size='15'>5</text><line class='sa-grid' x1='66.9' x2='847.8' y1='221.8' y2='221.8'/><text class='sa-text' x='57.9' y='227.8' text-anchor='end' font-size='15'>10</text><line class='sa-grid' x1='66.9' x2='847.8' y1='156.7' y2='156.7'/><text class='sa-text' x='57.9' y='162.7' text-anchor='end' font-size='15'>15</text><line class='sa-grid' x1='66.9' x2='847.8' y1='91.6' y2='91.6'/><text class='sa-text' x='57.9' y='97.6' text-anchor='end' font-size='15'>20</text><line class='sa-grid' x1='66.9' x2='847.8' y1='26.5' y2='26.5'/><text class='sa-text' x='57.9' y='32.5' text-anchor='end' font-size='15'>25</text><rect class='sa-frame-line' x='66.9' y='16.2' width='780.9' height='351.8'/><text class='sa-text' x='457.4' y='420.0' text-anchor='middle' font-size='15'>Feature Selection Time (s) per 1k Features</text><text class='sa-text' transform='translate(21.0 192.1) rotate(-90)' text-anchor='middle' font-size='15'>Improvement over Random (%)</text><defs><marker id='sa-arrow' viewBox='0 0 10 10' refX='9' refY='5' markerWidth='7' markerHeight='7' orient='auto-start-reverse'><path d='M0,0 L10,5 L0,10' fill='none' stroke='#009E73' stroke-width='1.6'/></marker></defs><line x1='109.5' y1='51.9' x2='68.9' y2='18.2' stroke='#009E73' stroke-width='2.2' marker-end='url(#sa-arrow)'/><text x='82.5' y='75.9' fill='#009E73' font-size='18' font-weight='700'>optimal</text><polyline points='102.4,352.0 344.2,54.5 523.5,32.2' fill='none' stroke='#D55E00' stroke-width='3.0'/><circle class='sa-point' cx='523.5' cy='32.2' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='530.9' cy='50.5' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='344.2' cy='54.5' r='6.8' fill='#D55E00' stroke-width='1'/><circle cx='698.4' cy='58.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='480.3' cy='66.1' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='534.6' cy='68.0' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='696.5' cy='69.7' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='491.8' cy='104.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='474.6' cy='112.9' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='812.3' cy='121.4' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='804.9' cy='140.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='746.8' cy='196.3' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle cx='811.4' cy='242.8' r='5.8' fill='#d3d3d3' fill-opacity='0.8' stroke='#808080' stroke-width='1'/><circle class='sa-point' cx='102.4' cy='352.0' r='6.8' fill='#D55E00' stroke-width='1'/><line class='sa-ink' x1='523.5' y1='32.2' x2='642.5' y2='25.6' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='642.5' y='25.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>RFImportance</text><line class='sa-ink' x1='530.9' y1='50.5' x2='638.0' y2='56.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='638.0' y='56.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MI</text><line class='sa-ink' x1='344.2' y1='54.5' x2='294.4' y2='62.5' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='294.4' y='62.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>F-Test</text><line class='sa-ink' x1='698.4' y1='58.4' x2='777.5' y2='52.1' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='777.5' y='52.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>ElasticNet</text><text class='sa-dim sa-halo' x='466.5' y='51.9' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>CART</text><line class='sa-ink' x1='534.6' y1='68.0' x2='569.2' y2='81.7' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='569.2' y='81.7' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>mRMR</text><line class='sa-ink' x1='696.5' y1='69.7' x2='734.7' y2='83.4' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='734.7' y='83.4' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>Lasso</text><line class='sa-ink' x1='491.8' y1='104.4' x2='376.5' y2='91.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='376.5' y='91.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>MarkovBlanket</text><line class='sa-ink' x1='474.6' y1='112.9' x2='545.7' y2='127.2' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='545.7' y='127.2' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>GainRatio</text><text class='sa-dim sa-halo' x='825.5' y='107.1' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SFS</text><line class='sa-ink' x1='804.9' y1='140.3' x2='739.6' y2='154.6' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='739.6' y='154.6' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>(R)ReliefF</text><line class='sa-ink' x1='746.8' y1='196.3' x2='714.0' y2='210.0' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='714.0' y='210.0' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>LOCO</text><line class='sa-ink' x1='811.4' y1='242.8' x2='786.9' y2='256.5' stroke-width='1' opacity='0.6'/><text class='sa-dim sa-halo' x='786.9' y='256.5' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='400'>SBE</text><line class='sa-ink' x1='102.4' y1='352.0' x2='156.2' y2='337.8' stroke-width='1' opacity='0.6'/><text class='sa-text sa-halo' x='156.2' y='337.8' text-anchor='middle' dominant-baseline='central' font-size='18' font-weight='700'>Random</text><rect class='sa-legend' x='610.8' y='296.0' width='225.0' height='60.0' rx='4'/><circle stroke='#808080' cx='631.8' cy='314.0' r='6.0' fill='#d3d3d3'/><text class='sa-text' x='649.8' y='320.0' font-size='15'>Dominated Methods</text><circle class='sa-point' cx='631.8' cy='338.0' r='6.0' fill='#D55E00'/><text class='sa-text' x='649.8' y='344.0' font-size='15'>Pareto Front</text><circle class='sa-hit' cx='523.5' cy='32.2' r='10.5' data-tip='RFImportance (on the Pareto front) Improvement over Random: 24.6% Selection time per 1k features: 38.8 s Datasets: 71 Next to MI: 23.2%, 46 s' aria-label='RFImportance (on the Pareto front) Improvement over Random: 24.6% Selection time per 1k features: 38.8 s Datasets: 71 Next to MI: 23.2%, 46 s'/><circle class='sa-hit' cx='530.9' cy='50.5' r='10.5' data-tip='MI Improvement over Random: 23.2% Selection time per 1k features: 46 s Datasets: 71 Next to RFImportance: 24.6%, 38.8 s Next to mRMR: 21.8%, 50 s' aria-label='MI Improvement over Random: 23.2% Selection time per 1k features: 46 s Datasets: 71 Next to RFImportance: 24.6%, 38.8 s Next to mRMR: 21.8%, 50 s'/><circle class='sa-hit' cx='344.2' cy='54.5' r='10.5' data-tip='F-Test (on the Pareto front) Improvement over Random: 22.9% Selection time per 1k features: 0.647 s Datasets: 71' aria-label='F-Test (on the Pareto front) Improvement over Random: 22.9% Selection time per 1k features: 0.647 s Datasets: 71'/><circle class='sa-hit' cx='698.4' cy='58.4' r='10.5' data-tip='ElasticNet Improvement over Random: 22.6% Selection time per 1k features: 2.1e+03 s Datasets: 71 Next to Lasso: 21.7%, 2.01e+03 s' aria-label='ElasticNet Improvement over Random: 22.6% Selection time per 1k features: 2.1e+03 s Datasets: 71 Next to Lasso: 21.7%, 2.01e+03 s'/><circle class='sa-hit' cx='480.3' cy='66.1' r='10.5' data-tip='CART Improvement over Random: 22.0% Selection time per 1k features: 14.5 s Datasets: 71' aria-label='CART Improvement over Random: 22.0% Selection time per 1k features: 14.5 s Datasets: 71'/><circle class='sa-hit' cx='534.6' cy='68.0' r='10.5' data-tip='mRMR Improvement over Random: 21.8% Selection time per 1k features: 50 s Datasets: 71 Next to MI: 23.2%, 46 s' aria-label='mRMR Improvement over Random: 21.8% Selection time per 1k features: 50 s Datasets: 71 Next to MI: 23.2%, 46 s'/><circle class='sa-hit' cx='696.5' cy='69.7' r='10.5' data-tip='Lasso Improvement over Random: 21.7% Selection time per 1k features: 2.01e+03 s Datasets: 71 Next to ElasticNet: 22.6%, 2.1e+03 s' aria-label='Lasso Improvement over Random: 21.7% Selection time per 1k features: 2.01e+03 s Datasets: 71 Next to ElasticNet: 22.6%, 2.1e+03 s'/><circle class='sa-hit' cx='491.8' cy='104.4' r='10.5' data-tip='MarkovBlanket Improvement over Random: 19.0% Selection time per 1k features: 18.8 s Datasets: 71 Next to GainRatio: 18.4%, 12.7 s' aria-label='MarkovBlanket Improvement over Random: 19.0% Selection time per 1k features: 18.8 s Datasets: 71 Next to GainRatio: 18.4%, 12.7 s'/><circle class='sa-hit' cx='474.6' cy='112.9' r='10.5' data-tip='GainRatio Improvement over Random: 18.4% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 19.0%, 18.8 s' aria-label='GainRatio Improvement over Random: 18.4% Selection time per 1k features: 12.7 s Datasets: 71 Next to MarkovBlanket: 19.0%, 18.8 s'/><circle class='sa-hit' cx='812.3' cy='121.4' r='10.5' data-tip='SFS Improvement over Random: 17.7% Selection time per 1k features: 2.83e+04 s Datasets: 71 Next to (R)ReliefF: 16.3%, 2.39e+04 s' aria-label='SFS Improvement over Random: 17.7% Selection time per 1k features: 2.83e+04 s Datasets: 71 Next to (R)ReliefF: 16.3%, 2.39e+04 s'/><circle class='sa-hit' cx='804.9' cy='140.3' r='10.5' data-tip='(R)ReliefF Improvement over Random: 16.3% Selection time per 1k features: 2.39e+04 s Datasets: 71 Next to SFS: 17.7%, 2.83e+04 s' aria-label='(R)ReliefF Improvement over Random: 16.3% Selection time per 1k features: 2.39e+04 s Datasets: 71 Next to SFS: 17.7%, 2.83e+04 s'/><circle class='sa-hit' cx='746.8' cy='196.3' r='10.5' data-tip='LOCO Improvement over Random: 12.0% Selection time per 1k features: 6.34e+03 s Datasets: 71' aria-label='LOCO Improvement over Random: 12.0% Selection time per 1k features: 6.34e+03 s Datasets: 71'/><circle class='sa-hit' cx='811.4' cy='242.8' r='10.5' data-tip='SBE Improvement over Random: 8.4% Selection time per 1k features: 2.77e+04 s Datasets: 71' aria-label='SBE Improvement over Random: 8.4% Selection time per 1k features: 2.77e+04 s Datasets: 71'/><circle class='sa-hit' cx='102.4' cy='352.0' r='10.5' data-tip='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71' aria-label='Random (on the Pareto front) Improvement over Random: 0.0% Selection time per 1k features: 0.0026 s Datasets: 71'/></svg><script type='text/plain' id='sa-csv'>method,improvement,time_per_1k,n_datasets,pareto_optimal
|
| 36 |
RFImportance,24.5664,38.8189,71,True
|
| 37 |
MI,23.1632,45.9696,71,False
|
| 38 |
F-Test,22.8523,0.6474,71,True
|