diff --git a/data/lasagna/stability/tasks_all/binary_thr_003.html b/data/lasagna/stability/tasks_all/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..5e6979d2f421ff1a4d04db0d8be2f0a64079f8f3 --- /dev/null +++ b/data/lasagna/stability/tasks_all/binary_thr_003.html @@ -0,0 +1,1104 @@ + +SelectArena: stability (stability, all tasks)
Hover or tap a cell for its value.
F-TestLaplacianScoreMI(R)ReliefFGainRatioRFImportancemRMRElasticNetCARTMarkovBlanketLassoSFSLOCOSBERandom0.00.23.415.1217.42799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/stability/tasks_all/overlay_top_3.html b/data/lasagna/stability/tasks_all/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..644e1994d6c081ae48bd6df6937cc70d03a02c7a --- /dev/null +++ b/data/lasagna/stability/tasks_all/overlay_top_3.html @@ -0,0 +1,1104 @@ + +SelectArena: stability (stability, all tasks)
Hover or tap a cell for its value.
F-TestLaplacianScoreMI(R)ReliefFGainRatioRFImportancemRMRElasticNetCARTMarkovBlanketLassoSFSLOCOSBERandom0.00.23.415.1217.42799.7Events per Variable0.00.20.40.60.81.0stableunstable diff --git a/data/lasagna/stability/tasks_all/plain.html b/data/lasagna/stability/tasks_all/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..cf8bcb51361bf8cce946170ba267df06060a89cf --- /dev/null +++ b/data/lasagna/stability/tasks_all/plain.html @@ -0,0 +1,1104 @@ + +SelectArena: stability (stability, all tasks)
Hover or tap a cell for its value.
F-TestLaplacianScoreMI(R)ReliefFGainRatioRFImportancemRMRElasticNetCARTMarkovBlanketLassoSFSLOCOSBERandom0.00.23.415.1217.42799.7Events per Variable0.00.20.40.60.81.0stableunstable diff --git a/data/lasagna/stability/tasks_binary/binary_thr_003.html b/data/lasagna/stability/tasks_binary/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..b50b2458af4dbfeae42c7c70700c21039ad9edfe --- /dev/null +++ b/data/lasagna/stability/tasks_binary/binary_thr_003.html @@ -0,0 +1,807 @@ + +SelectArena: stability (stability, binary tasks)
Hover or tap a cell for its value.
F-TestLaplacianScoreGainRatio(R)ReliefFRFImportanceMImRMRElasticNetMarkovBlanketCARTLassoSFSLOCOSBERandom0.00.12.48.240.0406.82799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/stability/tasks_binary/overlay_top_3.html b/data/lasagna/stability/tasks_binary/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..f1610d2b76e8d5ea1b31e231d18df2a80ca0a559 --- /dev/null +++ b/data/lasagna/stability/tasks_binary/overlay_top_3.html @@ -0,0 +1,807 @@ + +SelectArena: stability (stability, binary tasks)
Hover or tap a cell for its value.
F-TestLaplacianScoreGainRatio(R)ReliefFRFImportanceMImRMRElasticNetMarkovBlanketCARTLassoSFSLOCOSBERandom0.00.12.48.240.0406.82799.7Events per Variable0.00.20.40.60.81.0stableunstable diff --git a/data/lasagna/stability/tasks_binary/plain.html b/data/lasagna/stability/tasks_binary/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..6bb64a8bce5a146ebb53c5a1b9631a308c956b11 --- /dev/null +++ b/data/lasagna/stability/tasks_binary/plain.html @@ -0,0 +1,807 @@ + +SelectArena: stability (stability, binary tasks)
Hover or tap a cell for its value.
F-TestLaplacianScoreGainRatio(R)ReliefFRFImportanceMImRMRElasticNetMarkovBlanketCARTLassoSFSLOCOSBERandom0.00.12.48.240.0406.82799.7Events per Variable0.00.20.40.60.81.0stableunstable diff --git a/data/lasagna/stability/tasks_classification/binary_thr_003.html b/data/lasagna/stability/tasks_classification/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..802ae58b289c18ddde76b5caaa22a8b20d7d369b --- /dev/null +++ b/data/lasagna/stability/tasks_classification/binary_thr_003.html @@ -0,0 +1,980 @@ + +SelectArena: stability (stability, classification tasks)
Hover or tap a cell for its value.
F-TestGainRatio(R)ReliefFLaplacianScoreRFImportanceMImRMRElasticNetMarkovBlanketCARTLassoSFSLOCOSBERandom0.00.11.78.242.92799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/stability/tasks_classification/overlay_top_3.html b/data/lasagna/stability/tasks_classification/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..54ed165cd54d016fd40a453cf00ab3a3b64c4827 --- /dev/null +++ b/data/lasagna/stability/tasks_classification/overlay_top_3.html @@ -0,0 +1,980 @@ + +SelectArena: stability (stability, classification tasks)
Hover or tap a cell for its value.
F-TestGainRatio(R)ReliefFLaplacianScoreRFImportanceMImRMRElasticNetMarkovBlanketCARTLassoSFSLOCOSBERandom0.00.11.78.242.92799.7Events per Variable0.00.20.40.60.81.0stableunstable diff --git a/data/lasagna/stability/tasks_classification/plain.html b/data/lasagna/stability/tasks_classification/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..e63b3b58d8b5d6a40d3387e028d54e72417b23d2 --- /dev/null +++ b/data/lasagna/stability/tasks_classification/plain.html @@ -0,0 +1,980 @@ + +SelectArena: stability (stability, classification tasks)
Hover or tap a cell for its value.
F-TestGainRatio(R)ReliefFLaplacianScoreRFImportanceMImRMRElasticNetMarkovBlanketCARTLassoSFSLOCOSBERandom0.00.11.78.242.92799.7Events per Variable0.00.20.40.60.81.0stableunstable diff --git a/data/lasagna/stability/tasks_multiclass/binary_thr_003.html b/data/lasagna/stability/tasks_multiclass/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..06f83199dfe613bcee8c6b8b610a8ba7bc2a4680 --- /dev/null +++ b/data/lasagna/stability/tasks_multiclass/binary_thr_003.html @@ -0,0 +1,319 @@ + +SelectArena: stability (stability, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatioRFImportance(R)ReliefFmRMRLaplacianScoreElasticNetMarkovBlanketLassoCARTSFSLOCOSBERandom0.00.12.522.11723.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/stability/tasks_multiclass/overlay_top_3.html b/data/lasagna/stability/tasks_multiclass/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..30cb6d7ddc2ddc142ff26f03fe7cacf932820c04 --- /dev/null +++ b/data/lasagna/stability/tasks_multiclass/overlay_top_3.html @@ -0,0 +1,319 @@ + +SelectArena: stability (stability, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatioRFImportance(R)ReliefFmRMRLaplacianScoreElasticNetMarkovBlanketLassoCARTSFSLOCOSBERandom0.00.12.522.11723.7Events per Variable0.00.20.40.60.81.0stableunstable diff --git a/data/lasagna/stability/tasks_multiclass/plain.html b/data/lasagna/stability/tasks_multiclass/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..eb3dc184406fea537cf3fb4f129576b3e625e532 --- /dev/null +++ b/data/lasagna/stability/tasks_multiclass/plain.html @@ -0,0 +1,319 @@ + +SelectArena: stability (stability, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatioRFImportance(R)ReliefFmRMRLaplacianScoreElasticNetMarkovBlanketLassoCARTSFSLOCOSBERandom0.00.12.522.11723.7Events per Variable0.00.20.40.60.81.0stableunstable diff --git a/data/lasagna/stability/tasks_regression/binary_thr_003.html b/data/lasagna/stability/tasks_regression/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..07e1cbd3c4f695af12331de67e0dfc580c901b54 --- /dev/null +++ b/data/lasagna/stability/tasks_regression/binary_thr_003.html @@ -0,0 +1,270 @@ + +SelectArena: stability (stability, regression tasks)
Hover or tap a cell for its value.
MILaplacianScoreF-TestRFImportance(R)ReliefFGainRatiomRMRCARTElasticNetLassoMarkovBlanketSFSLOCOSBERandom5.619.4259.3541.4918.41448.6Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/stability/tasks_regression/overlay_top_3.html b/data/lasagna/stability/tasks_regression/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..b6f83b4926dbc88433a743079405565f0f7b2de7 --- /dev/null +++ b/data/lasagna/stability/tasks_regression/overlay_top_3.html @@ -0,0 +1,270 @@ + +SelectArena: stability (stability, regression tasks)
Hover or tap a cell for its value.
MILaplacianScoreF-TestRFImportance(R)ReliefFGainRatiomRMRCARTElasticNetLassoMarkovBlanketSFSLOCOSBERandom5.619.4259.3541.4918.41448.6Events per Variable0.00.20.40.60.81.0stableunstable diff --git a/data/lasagna/stability/tasks_regression/plain.html b/data/lasagna/stability/tasks_regression/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..d8de351399215dd3700acc15f2960a524f50b743 --- /dev/null +++ b/data/lasagna/stability/tasks_regression/plain.html @@ -0,0 +1,270 @@ + +SelectArena: stability (stability, regression tasks)
Hover or tap a cell for its value.
MILaplacianScoreF-TestRFImportance(R)ReliefFGainRatiomRMRCARTElasticNetLassoMarkovBlanketSFSLOCOSBERandom5.619.4259.3541.4918.41448.6Events per Variable0.00.20.40.60.81.0stableunstable diff --git a/data/lasagna/validity/noise_0.5/tasks_all/binary_thr_003.html b/data/lasagna/validity/noise_0.5/tasks_all/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..25a97b01a63c93880ba7d2e2bff95c70a361ca4b --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_all/binary_thr_003.html @@ -0,0 +1,1164 @@ + +SelectArena: validity (noise 0.5, all tasks)
Hover or tap a cell for its value.
F-TestGainRatioMILaplacianScoreElasticNetLasso(R)ReliefFSFSRFImportancemRMRLOCOSBECARTRandomMarkovBlanket0.00.23.415.1217.42799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_0.5/tasks_all/overlay_top_3.html b/data/lasagna/validity/noise_0.5/tasks_all/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..7503926872ef4922d2766caa2b8f7a418fd68acf --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_all/overlay_top_3.html @@ -0,0 +1,1164 @@ + +SelectArena: validity (noise 0.5, all tasks)
Hover or tap a cell for its value.
F-TestGainRatioMILaplacianScoreElasticNetLasso(R)ReliefFSFSRFImportancemRMRLOCOSBECARTRandomMarkovBlanket0.00.23.415.1217.42799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.5/tasks_all/plain.html b/data/lasagna/validity/noise_0.5/tasks_all/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..bc7a880d1dbfae00428dde20c3f95e9725831433 --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_all/plain.html @@ -0,0 +1,1164 @@ + +SelectArena: validity (noise 0.5, all tasks)
Hover or tap a cell for its value.
F-TestGainRatioMILaplacianScoreElasticNetLasso(R)ReliefFSFSRFImportancemRMRLOCOSBECARTRandomMarkovBlanket0.00.23.415.1217.42799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.5/tasks_binary/binary_thr_003.html b/data/lasagna/validity/noise_0.5/tasks_binary/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..ed6dd664823c12240e66b3f94599918b5efd3a1e --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_binary/binary_thr_003.html @@ -0,0 +1,856 @@ + +SelectArena: validity (noise 0.5, binary tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFSFSRFImportancemRMRSBELOCORandomCARTMarkovBlanket0.00.12.48.240.0406.82799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_0.5/tasks_binary/overlay_top_3.html b/data/lasagna/validity/noise_0.5/tasks_binary/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..37ee3973c8223d8ceaed06b0aafbcc7ca5413191 --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_binary/overlay_top_3.html @@ -0,0 +1,856 @@ + +SelectArena: validity (noise 0.5, binary tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFSFSRFImportancemRMRSBELOCORandomCARTMarkovBlanket0.00.12.48.240.0406.82799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.5/tasks_binary/plain.html b/data/lasagna/validity/noise_0.5/tasks_binary/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..7ad4068ffa346b767d2f40319c1680823b29eee5 --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_binary/plain.html @@ -0,0 +1,856 @@ + +SelectArena: validity (noise 0.5, binary tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFSFSRFImportancemRMRSBELOCORandomCARTMarkovBlanket0.00.12.48.240.0406.82799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.5/tasks_classification/binary_thr_003.html b/data/lasagna/validity/noise_0.5/tasks_classification/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..24643785d022cbf122ae583b34a583fe5d4ae13c --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_classification/binary_thr_003.html @@ -0,0 +1,1036 @@ + +SelectArena: validity (noise 0.5, classification tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFSFSRFImportancemRMRLOCOSBECARTRandomMarkovBlanket0.00.11.78.242.92799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_0.5/tasks_classification/overlay_top_3.html b/data/lasagna/validity/noise_0.5/tasks_classification/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..bc559c5910c52b93051dfea6b4eba3144d755fb9 --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_classification/overlay_top_3.html @@ -0,0 +1,1036 @@ + +SelectArena: validity (noise 0.5, classification tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFSFSRFImportancemRMRLOCOSBECARTRandomMarkovBlanket0.00.11.78.242.92799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.5/tasks_classification/plain.html b/data/lasagna/validity/noise_0.5/tasks_classification/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..ad586b4939b79dcf0e9fedca1e511d30262bef8c --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_classification/plain.html @@ -0,0 +1,1036 @@ + +SelectArena: validity (noise 0.5, classification tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFSFSRFImportancemRMRLOCOSBECARTRandomMarkovBlanket0.00.11.78.242.92799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.5/tasks_multiclass/binary_thr_003.html b/data/lasagna/validity/noise_0.5/tasks_multiclass/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..444128b35d54aeff68d99b07a82051b1a30a2268 --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_multiclass/binary_thr_003.html @@ -0,0 +1,326 @@ + +SelectArena: validity (noise 0.5, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatio(R)ReliefFElasticNetLassomRMRLaplacianScoreRFImportanceCARTSFSLOCOSBERandomMarkovBlanket0.00.12.522.11723.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_0.5/tasks_multiclass/overlay_top_3.html b/data/lasagna/validity/noise_0.5/tasks_multiclass/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..99373854654ad0206a38d2b25913e738b103fa34 --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_multiclass/overlay_top_3.html @@ -0,0 +1,326 @@ + +SelectArena: validity (noise 0.5, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatio(R)ReliefFElasticNetLassomRMRLaplacianScoreRFImportanceCARTSFSLOCOSBERandomMarkovBlanket0.00.12.522.11723.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.5/tasks_multiclass/plain.html b/data/lasagna/validity/noise_0.5/tasks_multiclass/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..2ad5e1e01d132e0c73a5af17198a14aef7bcf7a9 --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_multiclass/plain.html @@ -0,0 +1,326 @@ + +SelectArena: validity (noise 0.5, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatio(R)ReliefFElasticNetLassomRMRLaplacianScoreRFImportanceCARTSFSLOCOSBERandomMarkovBlanket0.00.12.522.11723.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.5/tasks_regression/binary_thr_003.html b/data/lasagna/validity/noise_0.5/tasks_regression/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..8230298ed5b73a5ddbca5aaa496fdf1c565ab816 --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_regression/binary_thr_003.html @@ -0,0 +1,274 @@ + +SelectArena: validity (noise 0.5, regression tasks)
Hover or tap a cell for its value.
F-TestMI(R)ReliefFElasticNetLaplacianScoreLassoSFSLOCOGainRatioMarkovBlanketRFImportanceSBECARTmRMRRandom5.619.4259.3541.4918.41448.6Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_0.5/tasks_regression/overlay_top_3.html b/data/lasagna/validity/noise_0.5/tasks_regression/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..2794c141749828f143384e00170302a275999756 --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_regression/overlay_top_3.html @@ -0,0 +1,274 @@ + +SelectArena: validity (noise 0.5, regression tasks)
Hover or tap a cell for its value.
F-TestMI(R)ReliefFElasticNetLaplacianScoreLassoSFSLOCOGainRatioMarkovBlanketRFImportanceSBECARTmRMRRandom5.619.4259.3541.4918.41448.6Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.5/tasks_regression/plain.html b/data/lasagna/validity/noise_0.5/tasks_regression/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..8a207986ee4dc5d5ea438f271fb697e5006af64d --- /dev/null +++ b/data/lasagna/validity/noise_0.5/tasks_regression/plain.html @@ -0,0 +1,274 @@ + +SelectArena: validity (noise 0.5, regression tasks)
Hover or tap a cell for its value.
F-TestMI(R)ReliefFElasticNetLaplacianScoreLassoSFSLOCOGainRatioMarkovBlanketRFImportanceSBECARTmRMRRandom5.619.4259.3541.4918.41448.6Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.75/tasks_all/binary_thr_003.html b/data/lasagna/validity/noise_0.75/tasks_all/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..9263ad30998eada04cc324638c3969d90b736100 --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_all/binary_thr_003.html @@ -0,0 +1,1164 @@ + +SelectArena: validity (noise 0.75, all tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNetLasso(R)ReliefFRFImportanceSFSmRMRCARTLOCOSBERandomMarkovBlanket0.00.23.415.1217.42799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_0.75/tasks_all/overlay_top_3.html b/data/lasagna/validity/noise_0.75/tasks_all/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..6a731150d62a274d5c56d3d6c9637be29ce044e7 --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_all/overlay_top_3.html @@ -0,0 +1,1164 @@ + +SelectArena: validity (noise 0.75, all tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNetLasso(R)ReliefFRFImportanceSFSmRMRCARTLOCOSBERandomMarkovBlanket0.00.23.415.1217.42799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.75/tasks_all/plain.html b/data/lasagna/validity/noise_0.75/tasks_all/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..d9e92b039d597b1c3bbd3c84b2886366597f253e --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_all/plain.html @@ -0,0 +1,1164 @@ + +SelectArena: validity (noise 0.75, all tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNetLasso(R)ReliefFRFImportanceSFSmRMRCARTLOCOSBERandomMarkovBlanket0.00.23.415.1217.42799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.75/tasks_binary/binary_thr_003.html b/data/lasagna/validity/noise_0.75/tasks_binary/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..345c227a7688756fafc9bf7dc10bbd7da64e28b2 --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_binary/binary_thr_003.html @@ -0,0 +1,856 @@ + +SelectArena: validity (noise 0.75, binary tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFSFSRFImportancemRMRSBELOCOCARTRandomMarkovBlanket0.00.12.48.240.0406.82799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_0.75/tasks_binary/overlay_top_3.html b/data/lasagna/validity/noise_0.75/tasks_binary/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..afd0093bb1070c90c834c494ccc13d5858c818c6 --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_binary/overlay_top_3.html @@ -0,0 +1,856 @@ + +SelectArena: validity (noise 0.75, binary tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFSFSRFImportancemRMRSBELOCOCARTRandomMarkovBlanket0.00.12.48.240.0406.82799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.75/tasks_binary/plain.html b/data/lasagna/validity/noise_0.75/tasks_binary/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..1b6fbad8962bc905aa574bd1f347cf3799485e49 --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_binary/plain.html @@ -0,0 +1,856 @@ + +SelectArena: validity (noise 0.75, binary tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFSFSRFImportancemRMRSBELOCOCARTRandomMarkovBlanket0.00.12.48.240.0406.82799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.75/tasks_classification/binary_thr_003.html b/data/lasagna/validity/noise_0.75/tasks_classification/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..d02622527f1aaee156715b420644c0c606d9bd4a --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_classification/binary_thr_003.html @@ -0,0 +1,1036 @@ + +SelectArena: validity (noise 0.75, classification tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNetLasso(R)ReliefFRFImportancemRMRSFSCARTSBELOCORandomMarkovBlanket0.00.11.78.242.92799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_0.75/tasks_classification/overlay_top_3.html b/data/lasagna/validity/noise_0.75/tasks_classification/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..6ba385db9febac34c30a72daa279b0b35a899c2c --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_classification/overlay_top_3.html @@ -0,0 +1,1036 @@ + +SelectArena: validity (noise 0.75, classification tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNetLasso(R)ReliefFRFImportancemRMRSFSCARTSBELOCORandomMarkovBlanket0.00.11.78.242.92799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.75/tasks_classification/plain.html b/data/lasagna/validity/noise_0.75/tasks_classification/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..077fdaf7871e98e6558fda966f08d1f2e56591a1 --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_classification/plain.html @@ -0,0 +1,1036 @@ + +SelectArena: validity (noise 0.75, classification tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNetLasso(R)ReliefFRFImportancemRMRSFSCARTSBELOCORandomMarkovBlanket0.00.11.78.242.92799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.75/tasks_multiclass/binary_thr_003.html b/data/lasagna/validity/noise_0.75/tasks_multiclass/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..65c38453072457b2fa1dd9ce1deb95e2991ee3ba --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_multiclass/binary_thr_003.html @@ -0,0 +1,326 @@ + +SelectArena: validity (noise 0.75, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatio(R)ReliefFElasticNetLassomRMRLaplacianScoreRFImportanceCARTSFSLOCOSBERandomMarkovBlanket0.00.12.522.11723.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_0.75/tasks_multiclass/overlay_top_3.html b/data/lasagna/validity/noise_0.75/tasks_multiclass/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..08d28c75388d300e5d85876213c8c05fb1d4e78e --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_multiclass/overlay_top_3.html @@ -0,0 +1,326 @@ + +SelectArena: validity (noise 0.75, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatio(R)ReliefFElasticNetLassomRMRLaplacianScoreRFImportanceCARTSFSLOCOSBERandomMarkovBlanket0.00.12.522.11723.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.75/tasks_multiclass/plain.html b/data/lasagna/validity/noise_0.75/tasks_multiclass/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..09691c8fbdaeca2c92b1505825bf281d035603e3 --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_multiclass/plain.html @@ -0,0 +1,326 @@ + +SelectArena: validity (noise 0.75, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatio(R)ReliefFElasticNetLassomRMRLaplacianScoreRFImportanceCARTSFSLOCOSBERandomMarkovBlanket0.00.12.522.11723.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.75/tasks_regression/binary_thr_003.html b/data/lasagna/validity/noise_0.75/tasks_regression/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..1b5adcd14bd484459ee6a8f7e2e64209d971e23f --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_regression/binary_thr_003.html @@ -0,0 +1,274 @@ + +SelectArena: validity (noise 0.75, regression tasks)
Hover or tap a cell for its value.
MIF-Test(R)ReliefFElasticNetLaplacianScoreLassoGainRatioMarkovBlanketRFImportanceLOCOCARTSFSSBEmRMRRandom5.619.4259.3541.4918.41448.6Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_0.75/tasks_regression/overlay_top_3.html b/data/lasagna/validity/noise_0.75/tasks_regression/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..506cac34c70b591843fe97f5682682bb6200bf52 --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_regression/overlay_top_3.html @@ -0,0 +1,274 @@ + +SelectArena: validity (noise 0.75, regression tasks)
Hover or tap a cell for its value.
MIF-Test(R)ReliefFElasticNetLaplacianScoreLassoGainRatioMarkovBlanketRFImportanceLOCOCARTSFSSBEmRMRRandom5.619.4259.3541.4918.41448.6Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_0.75/tasks_regression/plain.html b/data/lasagna/validity/noise_0.75/tasks_regression/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..4def76fbb808710890a09ba2b56ef17800919e48 --- /dev/null +++ b/data/lasagna/validity/noise_0.75/tasks_regression/plain.html @@ -0,0 +1,274 @@ + +SelectArena: validity (noise 0.75, regression tasks)
Hover or tap a cell for its value.
MIF-Test(R)ReliefFElasticNetLaplacianScoreLassoGainRatioMarkovBlanketRFImportanceLOCOCARTSFSSBEmRMRRandom5.619.4259.3541.4918.41448.6Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_1/tasks_all/binary_thr_003.html b/data/lasagna/validity/noise_1/tasks_all/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..826a10c65d5e1c61e1bc7736032c8c64021b243a --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_all/binary_thr_003.html @@ -0,0 +1,1164 @@ + +SelectArena: validity (noise 1, all tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNet(R)ReliefFLassoRFImportancemRMRSFSCARTLOCOSBERandomMarkovBlanket0.00.23.415.1217.42799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_1/tasks_all/overlay_top_3.html b/data/lasagna/validity/noise_1/tasks_all/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..4fb1900f1c02ea74e922bcd110e840c6ca34b70d --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_all/overlay_top_3.html @@ -0,0 +1,1164 @@ + +SelectArena: validity (noise 1, all tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNet(R)ReliefFLassoRFImportancemRMRSFSCARTLOCOSBERandomMarkovBlanket0.00.23.415.1217.42799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_1/tasks_all/plain.html b/data/lasagna/validity/noise_1/tasks_all/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..6000978521d0a90fdf8bdd599f96e7e5047c29ae --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_all/plain.html @@ -0,0 +1,1164 @@ + +SelectArena: validity (noise 1, all tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNet(R)ReliefFLassoRFImportancemRMRSFSCARTLOCOSBERandomMarkovBlanket0.00.23.415.1217.42799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_1/tasks_binary/binary_thr_003.html b/data/lasagna/validity/noise_1/tasks_binary/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..5c4f50eb6dd23d11c9c3d0d90be78f411b2c5e12 --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_binary/binary_thr_003.html @@ -0,0 +1,856 @@ + +SelectArena: validity (noise 1, binary tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFRFImportancemRMRSFSLOCOCARTSBERandomMarkovBlanket0.00.12.48.240.0406.82799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_1/tasks_binary/overlay_top_3.html b/data/lasagna/validity/noise_1/tasks_binary/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..e9c37b67477ca706254be343146eac1c1c46b72e --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_binary/overlay_top_3.html @@ -0,0 +1,856 @@ + +SelectArena: validity (noise 1, binary tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFRFImportancemRMRSFSLOCOCARTSBERandomMarkovBlanket0.00.12.48.240.0406.82799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_1/tasks_binary/plain.html b/data/lasagna/validity/noise_1/tasks_binary/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..cbb6086866fe4497494b0f7d38bbc80a85244684 --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_binary/plain.html @@ -0,0 +1,856 @@ + +SelectArena: validity (noise 1, binary tasks)
Hover or tap a cell for its value.
GainRatioF-TestLaplacianScoreMIElasticNetLasso(R)ReliefFRFImportancemRMRSFSLOCOCARTSBERandomMarkovBlanket0.00.12.48.240.0406.82799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_1/tasks_classification/binary_thr_003.html b/data/lasagna/validity/noise_1/tasks_classification/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..6e8b04280fb25f52751fa75719fc902007b5fb14 --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_classification/binary_thr_003.html @@ -0,0 +1,1036 @@ + +SelectArena: validity (noise 1, classification tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNet(R)ReliefFLassoRFImportancemRMRSFSCARTLOCOSBERandomMarkovBlanket0.00.11.78.242.92799.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_1/tasks_classification/overlay_top_3.html b/data/lasagna/validity/noise_1/tasks_classification/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..ce7e33859e67929176461d6aa22960b39d2c1047 --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_classification/overlay_top_3.html @@ -0,0 +1,1036 @@ + +SelectArena: validity (noise 1, classification tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNet(R)ReliefFLassoRFImportancemRMRSFSCARTLOCOSBERandomMarkovBlanket0.00.11.78.242.92799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_1/tasks_classification/plain.html b/data/lasagna/validity/noise_1/tasks_classification/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..eb0a1ac7e0a7fa5a9e41f4df15a34be5009160f2 --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_classification/plain.html @@ -0,0 +1,1036 @@ + +SelectArena: validity (noise 1, classification tasks)
Hover or tap a cell for its value.
F-TestGainRatioLaplacianScoreMIElasticNet(R)ReliefFLassoRFImportancemRMRSFSCARTLOCOSBERandomMarkovBlanket0.00.11.78.242.92799.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_1/tasks_multiclass/binary_thr_003.html b/data/lasagna/validity/noise_1/tasks_multiclass/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..c57fcb640fbfb9787ac19d0af5c96a30dbf1d5b1 --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_multiclass/binary_thr_003.html @@ -0,0 +1,326 @@ + +SelectArena: validity (noise 1, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatio(R)ReliefFElasticNetLassomRMRLaplacianScoreRFImportanceCARTLOCOSFSSBERandomMarkovBlanket0.00.12.522.11723.7Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_1/tasks_multiclass/overlay_top_3.html b/data/lasagna/validity/noise_1/tasks_multiclass/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..af46ac796d22e0e140a416569107d424985dfdef --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_multiclass/overlay_top_3.html @@ -0,0 +1,326 @@ + +SelectArena: validity (noise 1, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatio(R)ReliefFElasticNetLassomRMRLaplacianScoreRFImportanceCARTLOCOSFSSBERandomMarkovBlanket0.00.12.522.11723.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_1/tasks_multiclass/plain.html b/data/lasagna/validity/noise_1/tasks_multiclass/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..459801462426ade473fe6b4f585882b939471335 --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_multiclass/plain.html @@ -0,0 +1,326 @@ + +SelectArena: validity (noise 1, multiclass tasks)
Hover or tap a cell for its value.
F-TestMIGainRatio(R)ReliefFElasticNetLassomRMRLaplacianScoreRFImportanceCARTLOCOSFSSBERandomMarkovBlanket0.00.12.522.11723.7Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_1/tasks_regression/binary_thr_003.html b/data/lasagna/validity/noise_1/tasks_regression/binary_thr_003.html new file mode 100644 index 0000000000000000000000000000000000000000..d954435df11aa6895e8c7607b4d764c35e4495ba --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_regression/binary_thr_003.html @@ -0,0 +1,274 @@ + +SelectArena: validity (noise 1, regression tasks)
Hover or tap a cell for its value.
MIF-Test(R)ReliefFElasticNetLaplacianScoreLassoGainRatioRFImportanceMarkovBlanketSFSCARTLOCOSBEmRMRRandom5.619.4259.3541.4918.41448.6Events per VariableWithin ThresholdOutside Threshold diff --git a/data/lasagna/validity/noise_1/tasks_regression/overlay_top_3.html b/data/lasagna/validity/noise_1/tasks_regression/overlay_top_3.html new file mode 100644 index 0000000000000000000000000000000000000000..5c44810f6db90935bf1441ef32ccde8d0f49c1f7 --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_regression/overlay_top_3.html @@ -0,0 +1,274 @@ + +SelectArena: validity (noise 1, regression tasks)
Hover or tap a cell for its value.
MIF-Test(R)ReliefFElasticNetLaplacianScoreLassoGainRatioRFImportanceMarkovBlanketSFSCARTLOCOSBEmRMRRandom5.619.4259.3541.4918.41448.6Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/lasagna/validity/noise_1/tasks_regression/plain.html b/data/lasagna/validity/noise_1/tasks_regression/plain.html new file mode 100644 index 0000000000000000000000000000000000000000..0a1361a4d6c2b86e081d1908e01a473b93f6f5d5 --- /dev/null +++ b/data/lasagna/validity/noise_1/tasks_regression/plain.html @@ -0,0 +1,274 @@ + +SelectArena: validity (noise 1, regression tasks)
Hover or tap a cell for its value.
MIF-Test(R)ReliefFElasticNetLaplacianScoreLassoGainRatioRFImportanceMarkovBlanketSFSCARTLOCOSBEmRMRRandom5.619.4259.3541.4918.41448.6Events per Variable0.00.20.40.60.81.0validinvalid diff --git a/data/subsets/tasks_all/models_all/budgets_0/pareto.html b/data/subsets/tasks_all/models_all/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..ed1d52e19980ba0f3ba177174c3298f005229dac --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, all models, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIF-TestCARTMarkovBlanketElasticNetmRMRLassoSFSGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_all/budgets_0/per_model.csv b/data/subsets/tasks_all/models_all/budgets_0/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..a7727042916070df163b73ad549f5ce7a38c3db5 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_0/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1343.0,1368.0,1,1352.0,1,1311.0,1,1335.0,1,1359.0,1 +2,CART,Non-search,Tree,1315.0,1347.0,2,1344.0,3,1265.0,3,1311.0,3,1317.0,3 +3,MarkovBlanket,Search,Backward search,1308.0,1325.0,3,1323.0,6,1257.0,5,1324.0,2,1319.0,2 +4,F-Test,Non-search,Correlation,1290.0,1280.0,7,1350.0,2,1262.0,4,1285.0,5,1282.0,4 +5,MI,Non-search,Information theory,1290.0,1292.0,6,1324.0,5,1278.0,2,1287.0,4,1273.0,6 +6,ElasticNet,Non-search,Regularization,1284.0,1304.0,5,1327.0,4,1234.0,8,1282.0,6,1281.0,5 +7,SFS,Search,Forward search,1270.0,1312.0,4,1295.0,8,1245.0,6,1257.0,8,1248.0,8 +8,mRMR,Non-search,Information theory,1255.0,1254.0,9,1301.0,7,1213.0,9,1268.0,7,1246.0,9 +9,Lasso,Non-search,Regularization,1250.0,1270.0,8,1291.0,9,1201.0,11,1245.0,9,1250.0,7 +10,GainRatio,Non-search,Information theory,1242.0,1234.0,10,1274.0,10,1236.0,7,1235.0,10,1237.0,10 +11,(R)ReliefF,Non-search,Distance,1206.0,1191.0,13,1226.0,11,1201.0,10,1203.0,11,1213.0,11 +12,SBE,Search,Backward search,1186.0,1233.0,11,1168.0,13,1151.0,12,1189.0,12,1194.0,13 +13,LOCO,Search,Leave-one-out,1176.0,1194.0,12,1197.0,12,1129.0,13,1162.0,13,1206.0,12 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,15,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,963.0,934.0,15,973.0,15,1003.0,14,929.0,15,966.0,15 diff --git a/data/subsets/tasks_all/models_all/budgets_0/per_model_table.html b/data/subsets/tasks_all/models_all/budgets_0/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..9a8116bfcfe89a3cc7fdd0eaf37d139671b79b91 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_0/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, all models, budget 1 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1343🥇1368🥇1352🥇1311🥇1335🥇1359
🥈2CART1315🥈1347🥉1344🥉1265🥉1311🥉1317
🥉3MarkovBlanket1308🥉132513231257🥈1324🥈1319
4F-Test12901280🥈1350126212851282
5MI129012921324🥈127812871273
6ElasticNet128413041327123412821281
7SFS127013121295124512571248
8mRMR125512541301121312681246
9Lasso125012701291120112451250
10GainRatio124212341274123612351237
11(R)ReliefF120611911226120112031213
12SBE118612331168115111891194
13LOCO117611941197112911621206
14Random100010001000100010001000
15LaplacianScore9639349731003929966
diff --git a/data/subsets/tasks_all/models_all/budgets_1/pareto.html b/data/subsets/tasks_all/models_all/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..e5170857efbd1d7e51ca17074ccc0d6e3a042888 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, all models, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTElasticNetF-TestLassoMarkovBlanketmRMRGainRatioSFS(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_all/budgets_1/per_model.csv b/data/subsets/tasks_all/models_all/budgets_1/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..2773d395541b1b464b3cbdcfcbfae8dda9b28434 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_1/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1366.0,1366.0,1,1365.0,3,1370.0,1,1349.0,1,1387.0,1 +2,ElasticNet,Non-search,Regularization,1346.0,1366.0,2,1405.0,1,1310.0,3,1335.0,3,1324.0,4 +3,CART,Non-search,Tree,1335.0,1355.0,3,1318.0,6,1316.0,2,1339.0,2,1356.0,2 +4,Lasso,Non-search,Regularization,1323.0,1338.0,5,1377.0,2,1296.0,4,1307.0,4,1309.0,7 +5,MI,Non-search,Information theory,1316.0,1325.0,6,1342.0,4,1293.0,5,1287.0,7,1341.0,3 +6,MarkovBlanket,Search,Backward search,1307.0,1340.0,4,1301.0,7,1290.0,6,1293.0,5,1320.0,5 +7,F-Test,Non-search,Correlation,1298.0,1289.0,10,1326.0,5,1272.0,7,1288.0,6,1320.0,6 +8,GainRatio,Non-search,Information theory,1273.0,1291.0,9,1280.0,9,1266.0,8,1260.0,9,1272.0,10 +9,mRMR,Non-search,Information theory,1268.0,1311.0,7,1296.0,8,1228.0,11,1236.0,11,1277.0,8 +10,(R)ReliefF,Non-search,Distance,1257.0,1275.0,11,1216.0,11,1254.0,9,1269.0,8,1276.0,9 +11,SFS,Search,Forward search,1256.0,1294.0,8,1267.0,10,1224.0,12,1253.0,10,1246.0,11 +12,SBE,Search,Backward search,1227.0,1259.0,12,1184.0,13,1243.0,10,1221.0,13,1231.0,13 +13,LOCO,Search,Leave-one-out,1219.0,1241.0,13,1199.0,12,1192.0,13,1226.0,12,1244.0,12 +14,LaplacianScore,Non-search,Distance,1025.0,995.0,15,1026.0,14,1078.0,14,1000.0,15,1014.0,14 +15,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,15,1000.0,15,1000.0,14,1000.0,15 diff --git a/data/subsets/tasks_all/models_all/budgets_1/per_model_table.html b/data/subsets/tasks_all/models_all/budgets_1/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..834e6eaafa304e2f1925c72637d7df9fe2ec8226 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_1/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, all models, budget 2 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1366🥇1366🥉1365🥇1370🥇1349🥇1387
🥈2ElasticNet1346🥈1366🥇1405🥉1310🥉13351324
🥉3CART1335🥉13551318🥈1316🥈1339🥈1356
4Lasso13231338🥈1377129613071309
5MI13161325134212931287🥉1341
6MarkovBlanket130713401301129012931320
7F-Test129812891326127212881320
8GainRatio127312911280126612601272
9mRMR126813111296122812361277
10(R)ReliefF125712751216125412691276
11SFS125612941267122412531246
12SBE122712591184124312211231
13LOCO121912411199119212261244
14LaplacianScore10259951026107810001014
15Random100010001000100010001000
diff --git a/data/subsets/tasks_all/models_all/budgets_2/pareto.html b/data/subsets/tasks_all/models_all/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..53c4658fb68446214636c5f6ebec048a464a32a0 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, all models, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTElasticNet(R)ReliefFLassoF-TestMarkovBlanketmRMRGainRatioSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_all/budgets_2/per_model.csv b/data/subsets/tasks_all/models_all/budgets_2/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..6786614064526168e10b271fc5bc323f140b6b57 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_2/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1325.0,1321.0,1,1263.0,3,1377.0,1,1336.0,1,1340.0,1 +2,CART,Non-search,Tree,1278.0,1283.0,2,1203.0,6,1302.0,2,1298.0,2,1311.0,2 +3,ElasticNet,Non-search,Regularization,1273.0,1252.0,5,1313.0,1,1259.0,4,1296.0,3,1256.0,4 +4,MI,Non-search,Information theory,1272.0,1282.0,3,1258.0,4,1283.0,3,1262.0,5,1285.0,3 +5,Lasso,Non-search,Regularization,1249.0,1268.0,4,1274.0,2,1236.0,7,1250.0,6,1224.0,8 +6,MarkovBlanket,Search,Backward search,1237.0,1248.0,6,1172.0,9,1252.0,5,1264.0,4,1253.0,5 +7,F-Test,Non-search,Correlation,1223.0,1212.0,8,1216.0,5,1239.0,6,1223.0,7,1231.0,7 +8,(R)ReliefF,Non-search,Distance,1210.0,1199.0,10,1168.0,11,1235.0,8,1218.0,8,1237.0,6 +9,GainRatio,Non-search,Information theory,1194.0,1205.0,9,1185.0,7,1190.0,10,1208.0,9,1186.0,11 +10,SBE,Search,Backward search,1186.0,1224.0,7,1088.0,13,1204.0,9,1198.0,10,1220.0,9 +11,mRMR,Non-search,Information theory,1181.0,1197.0,11,1173.0,8,1179.0,11,1166.0,13,1196.0,10 +12,SFS,Search,Forward search,1175.0,1193.0,12,1172.0,10,1164.0,12,1180.0,11,1169.0,13 +13,LOCO,Search,Leave-one-out,1157.0,1175.0,13,1109.0,12,1152.0,13,1177.0,12,1178.0,12 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,930.0,939.0,15,942.0,15,951.0,15,893.0,15,916.0,15 diff --git a/data/subsets/tasks_all/models_all/budgets_2/per_model_table.html b/data/subsets/tasks_all/models_all/budgets_2/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..d3f94c416713bcb553bf85a927b6411707375bac --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_2/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, all models, budget 3 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1325🥇1321🥉1263🥇1377🥇1336🥇1340
🥈2CART1278🥈12831203🥈1302🥈1298🥈1311
🥉3ElasticNet12731252🥇13131259🥉12961256
4MI1272🥉12821258🥉12831262🥉1285
5Lasso12491268🥈1274123612501224
6MarkovBlanket123712481172125212641253
7F-Test122312121216123912231231
8(R)ReliefF121011991168123512181237
9GainRatio119412051185119012081186
10SBE118612241088120411981220
11mRMR118111971173117911661196
12SFS117511931172116411801169
13LOCO115711751109115211771178
14Random100010001000100010001000
15LaplacianScore930939942951893916
diff --git a/data/subsets/tasks_all/models_all/budgets_3/pareto.html b/data/subsets/tasks_all/models_all/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..3b8746b3a3b30d8e2d7f0814f9fe3eff6282a16b --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, all models, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.5Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMImRMRF-TestElasticNet(R)ReliefFLassoGainRatioMarkovBlanketSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_all/budgets_3/per_model.csv b/data/subsets/tasks_all/models_all/budgets_3/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..60be44f275cdd335fdea77a4e73190d045b2300f --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_3/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1288.0,1273.0,1,1246.0,1,1320.0,1,1259.0,1,1350.0,1 +2,CART,Non-search,Tree,1232.0,1210.0,2,1162.0,6,1275.0,2,1253.0,2,1268.0,2 +3,MI,Non-search,Information theory,1218.0,1197.0,5,1213.0,4,1222.0,4,1229.0,3,1232.0,3 +4,ElasticNet,Non-search,Regularization,1212.0,1207.0,4,1237.0,2,1229.0,3,1208.0,4,1185.0,8 +5,Lasso,Non-search,Regularization,1192.0,1207.0,3,1223.0,3,1192.0,6,1160.0,10,1182.0,9 +6,F-Test,Non-search,Correlation,1189.0,1183.0,6,1186.0,5,1183.0,8,1176.0,7,1222.0,4 +7,(R)ReliefF,Non-search,Distance,1172.0,1145.0,8,1124.0,9,1206.0,5,1190.0,5,1199.0,6 +8,MarkovBlanket,Search,Backward search,1158.0,1128.0,9,1106.0,13,1188.0,7,1178.0,6,1192.0,7 +9,SBE,Search,Backward search,1155.0,1122.0,10,1121.0,10,1171.0,10,1162.0,8,1203.0,5 +10,mRMR,Non-search,Information theory,1155.0,1153.0,7,1152.0,7,1180.0,9,1143.0,12,1151.0,11 +11,GainRatio,Non-search,Information theory,1134.0,1115.0,12,1119.0,11,1147.0,11,1134.0,13,1158.0,10 +12,LOCO,Search,Leave-one-out,1127.0,1118.0,11,1108.0,12,1110.0,12,1162.0,9,1139.0,13 +13,SFS,Search,Forward search,1124.0,1098.0,13,1130.0,8,1105.0,13,1146.0,11,1142.0,12 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,929.0,878.0,15,977.0,15,943.0,15,905.0,15,932.0,15 diff --git a/data/subsets/tasks_all/models_all/budgets_3/per_model_table.html b/data/subsets/tasks_all/models_all/budgets_3/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..4ba257cd1f63951b76d3b4801322bd9e7f3ac061 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_3/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, all models, budget 4 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1288🥇1273🥇1246🥇1320🥇1259🥇1350
🥈2CART1232🥈12101162🥈1275🥈1253🥈1268
🥉3MI1218119712131222🥉1229🥉1232
4ElasticNet12121207🥈1237🥉122912081185
5Lasso1192🥉1207🥉1223119211601182
6F-Test118911831186118311761222
7(R)ReliefF117211451124120611901199
8MarkovBlanket115811281106118811781192
9SBE115511221121117111621203
10mRMR115511531152118011431151
11GainRatio113411151119114711341158
12LOCO112711181108111011621139
13SFS112410981130110511461142
14Random100010001000100010001000
15LaplacianScore929878977943905932
diff --git a/data/subsets/tasks_all/models_all/budgets_4/pareto.html b/data/subsets/tasks_all/models_all/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..d9add611cc14bc1210c9b7602f9b8c8b9b47f242 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, all models, budget 5
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestmRMRElasticNet(R)ReliefFCARTLassoMIMarkovBlanketGainRatioLOCOSBESFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_all/budgets_4/per_model.csv b/data/subsets/tasks_all/models_all/budgets_4/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..481fbe4f295c7a34de43b03df5b5bc044b45e6ab --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_4/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1250.0,1274.0,1,1220.0,1,1265.0,1,1198.0,1,1307.0,1 +2,CART,Non-search,Tree,1189.0,1177.0,2,1169.0,7,1217.0,2,1187.0,2,1200.0,4 +3,ElasticNet,Non-search,Regularization,1172.0,1171.0,3,1187.0,3,1176.0,5,1145.0,5,1185.0,5 +4,MI,Non-search,Information theory,1171.0,1160.0,5,1182.0,4,1181.0,4,1166.0,3,1167.0,6 +5,F-Test,Non-search,Correlation,1168.0,1154.0,6,1189.0,2,1171.0,6,1129.0,6,1204.0,2 +6,(R)ReliefF,Non-search,Distance,1165.0,1148.0,7,1138.0,8,1192.0,3,1149.0,4,1202.0,3 +7,Lasso,Non-search,Regularization,1140.0,1169.0,4,1170.0,6,1120.0,10,1101.0,8,1146.0,7 +8,mRMR,Non-search,Information theory,1131.0,1125.0,8,1176.0,5,1145.0,8,1084.0,10,1126.0,10 +9,MarkovBlanket,Search,Backward search,1115.0,1116.0,9,1114.0,10,1169.0,7,1067.0,11,1114.0,12 +10,SBE,Search,Backward search,1113.0,1112.0,10,1111.0,11,1116.0,11,1093.0,9,1137.0,8 +11,LOCO,Search,Leave-one-out,1097.0,1092.0,12,1089.0,13,1092.0,12,1110.0,7,1107.0,13 +12,GainRatio,Non-search,Information theory,1097.0,1091.0,13,1121.0,9,1128.0,9,1027.0,13,1119.0,11 +13,SFS,Search,Forward search,1092.0,1097.0,11,1102.0,12,1080.0,13,1055.0,12,1126.0,9 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,15,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,980.0,968.0,15,1042.0,14,947.0,15,967.0,15,970.0,15 diff --git a/data/subsets/tasks_all/models_all/budgets_4/per_model_table.html b/data/subsets/tasks_all/models_all/budgets_4/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..0d29cf65ad07e86750a9ef458eabc4da9f0c6d5c --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_4/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, all models, budget 5 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1250🥇1274🥇1220🥇1265🥇1198🥇1307
🥈2CART1189🥈11771169🥈1217🥈11871200
🥉3ElasticNet1172🥉1171🥉1187117611451185
4MI1171116011821181🥉11661167
5F-Test11681154🥈118911711129🥈1204
6(R)ReliefF116511481138🥉11921149🥉1202
7Lasso114011691170112011011146
8mRMR113111251176114510841126
9MarkovBlanket111511161114116910671114
10SBE111311121111111610931137
11LOCO109710921089109211101107
12GainRatio109710911121112810271119
13SFS109210971102108010551126
14Random100010001000100010001000
15LaplacianScore9809681042947967970
diff --git a/data/subsets/tasks_all/models_all/budgets_all/pareto.html b/data/subsets/tasks_all/models_all/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..16001d30a6cdfb003c6b33c0720d75dd389c6b80 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, all models, all budgets
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-TestElasticNetmRMRMarkovBlanketLassoGainRatio(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_all/budgets_all/per_model.csv b/data/subsets/tasks_all/models_all/budgets_all/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..9b2f8f79557fd372c1a604258ba2c41ec49564ae --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_all/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1310.0,1313.0,1,1282.0,2,1325.0,1,1289.0,1,1344.0,1 +2,CART,Non-search,Tree,1265.0,1267.0,2,1232.0,6,1272.0,2,1271.0,2,1286.0,2 +3,ElasticNet,Non-search,Regularization,1252.0,1252.0,3,1286.0,1,1239.0,4,1247.0,3,1242.0,5 +4,MI,Non-search,Information theory,1248.0,1244.0,4,1257.0,4,1248.0,3,1240.0,4,1255.0,3 +5,F-Test,Non-search,Correlation,1229.0,1217.0,7,1246.0,5,1222.0,6,1214.0,6,1248.0,4 +6,Lasso,Non-search,Regularization,1226.0,1244.0,5,1259.0,3,1206.0,8,1206.0,7,1218.0,8 +7,MarkovBlanket,Search,Backward search,1220.0,1224.0,6,1196.0,8,1228.0,5,1218.0,5,1235.0,6 +8,(R)ReliefF,Non-search,Distance,1198.0,1185.0,10,1169.0,11,1215.0,7,1200.0,8,1221.0,7 +9,mRMR,Non-search,Information theory,1193.0,1201.0,8,1213.0,7,1186.0,10,1174.0,9,1195.0,9 +10,GainRatio,Non-search,Information theory,1183.0,1181.0,12,1189.0,9,1191.0,9,1167.0,12,1191.0,11 +11,SFS,Search,Forward search,1178.0,1192.0,9,1186.0,10,1161.0,12,1172.0,10,1183.0,12 +12,SBE,Search,Backward search,1169.0,1184.0,11,1130.0,13,1174.0,11,1167.0,11,1193.0,10 +13,LOCO,Search,Leave-one-out,1151.0,1158.0,13,1135.0,12,1132.0,13,1162.0,13,1171.0,13 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,965.0,943.0,15,994.0,15,984.0,15,942.0,15,960.0,15 diff --git a/data/subsets/tasks_all/models_all/budgets_all/per_model_table.html b/data/subsets/tasks_all/models_all/budgets_all/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..6c421ba34f0894dde362efc9b81e112fb381d546 --- /dev/null +++ b/data/subsets/tasks_all/models_all/budgets_all/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, all models, all budgets · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1310🥇1313🥈1282🥇1325🥇1289🥇1344
🥈2CART1265🥈12671232🥈1272🥈1271🥈1286
🥉3ElasticNet1252🥉1252🥇12861239🥉12471242
4MI124812441257🥉12481240🥉1255
5F-Test122912171246122212141248
6Lasso12261244🥉1259120612061218
7MarkovBlanket122012241196122812181235
8(R)ReliefF119811851169121512001221
9mRMR119312011213118611741195
10GainRatio118311811189119111671191
11SFS117811921186116111721183
12SBE116911841130117411671193
13LOCO115111581135113211621171
14Random100010001000100010001000
15LaplacianScore965943994984942960
diff --git a/data/subsets/tasks_all/models_lightgbm/budgets_0/pareto.html b/data/subsets/tasks_all/models_lightgbm/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..8579afa527e0b9368106dbb1155715890421887a --- /dev/null +++ b/data/subsets/tasks_all/models_lightgbm/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LightGBM, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-TestMarkovBlanketmRMRElasticNetSFSLassoGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_lightgbm/budgets_1/pareto.html b/data/subsets/tasks_all/models_lightgbm/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..efb56bce9f74b41223e900719401c7600314974f --- /dev/null +++ b/data/subsets/tasks_all/models_lightgbm/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LightGBM, budget 2
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTElasticNetLassomRMRMarkovBlanketF-TestGainRatio(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_lightgbm/budgets_2/pareto.html b/data/subsets/tasks_all/models_lightgbm/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..68371df800f9509adcc8335c3de08821bdf8ce4c --- /dev/null +++ b/data/subsets/tasks_all/models_lightgbm/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LightGBM, budget 3
Hover or tap a method for its values.
10-210-110010110210310405101520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIElasticNetLassoMarkovBlanketmRMRF-TestGainRatio(R)ReliefFSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_lightgbm/budgets_3/pareto.html b/data/subsets/tasks_all/models_lightgbm/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..0c7d2093afd3007ddf40e02ffbbcb8fe38d10174 --- /dev/null +++ b/data/subsets/tasks_all/models_lightgbm/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LightGBM, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.5Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTF-TestMILassomRMRElasticNet(R)ReliefFGainRatioMarkovBlanketSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_lightgbm/budgets_4/pareto.html b/data/subsets/tasks_all/models_lightgbm/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..e12375674edb309cc736180ebab0b643ce7675ab --- /dev/null +++ b/data/subsets/tasks_all/models_lightgbm/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LightGBM, budget 5
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestmRMRCARTLassoMarkovBlanket(R)ReliefFElasticNetSBEGainRatioMISFSLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_lightgbm/budgets_all/pareto.html b/data/subsets/tasks_all/models_lightgbm/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..cb59115c2b56445b0d89d59a05a88de879e631f5 --- /dev/null +++ b/data/subsets/tasks_all/models_lightgbm/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LightGBM, all budgets
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMImRMRElasticNetF-TestMarkovBlanketLassoGainRatio(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_linearmodel/budgets_0/pareto.html b/data/subsets/tasks_all/models_linearmodel/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..f326bf36aeb68033b8fd4cfd324fe157d10a6979 --- /dev/null +++ b/data/subsets/tasks_all/models_linearmodel/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LinearModel, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalF-TestRFImportanceMICARTmRMRElasticNetMarkovBlanketLassoSFSGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_linearmodel/budgets_1/pareto.html b/data/subsets/tasks_all/models_linearmodel/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..1997060d21d2f4f56f6def5016d9ec21a30b7422 --- /dev/null +++ b/data/subsets/tasks_all/models_linearmodel/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LinearModel, budget 2
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIElasticNetF-TestLassomRMRCARTGainRatioMarkovBlanketSFS(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_linearmodel/budgets_2/pareto.html b/data/subsets/tasks_all/models_linearmodel/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..ac59a002cf2c891a3da45523e7d6474f5751dfbc --- /dev/null +++ b/data/subsets/tasks_all/models_linearmodel/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LinearModel, budget 3
Hover or tap a method for its values.
10-210-110010110210310405101520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIElasticNetLassoF-TestmRMR(R)ReliefFCARTGainRatioMarkovBlanketSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_linearmodel/budgets_3/pareto.html b/data/subsets/tasks_all/models_linearmodel/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..c31622c3e11fe3ea3291a56f0ca6340e832e0abc --- /dev/null +++ b/data/subsets/tasks_all/models_linearmodel/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LinearModel, budget 4
Hover or tap a method for its values.
10-210-11001011021031040246810121416Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceElasticNetMILassomRMRF-TestCARTGainRatio(R)ReliefFSFSMarkovBlanketLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_linearmodel/budgets_4/pareto.html b/data/subsets/tasks_all/models_linearmodel/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..ff44bf28b606993b8c5583b3f408819e54b70df4 --- /dev/null +++ b/data/subsets/tasks_all/models_linearmodel/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LinearModel, budget 5
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceElasticNetmRMRF-TestMIGainRatioMarkovBlanketCARTLasso(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_linearmodel/budgets_all/pareto.html b/data/subsets/tasks_all/models_linearmodel/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..f84c7871c55fd1ae1be0af35c19fc71735b3b9a6 --- /dev/null +++ b/data/subsets/tasks_all/models_linearmodel/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, LinearModel, all budgets
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIF-TestElasticNetCARTmRMRLassoMarkovBlanketGainRatioSFS(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_randomforest/budgets_0/pareto.html b/data/subsets/tasks_all/models_randomforest/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..49cbf0063d995a80303985cfde2e5f3e1fabbcf7 --- /dev/null +++ b/data/subsets/tasks_all/models_randomforest/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RandomForest, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalMIRFImportanceF-TestCARTMarkovBlanketElasticNetGainRatiomRMRLassoSFS(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_randomforest/budgets_1/pareto.html b/data/subsets/tasks_all/models_randomforest/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..f41eb387b91b589d10dcda0413207b53471f2953 --- /dev/null +++ b/data/subsets/tasks_all/models_randomforest/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RandomForest, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-TestMarkovBlanketElasticNetLassoGainRatio(R)ReliefFmRMRSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_randomforest/budgets_2/pareto.html b/data/subsets/tasks_all/models_randomforest/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..18d06087fc683e2cb39dd8f54a8b5d12ef6ed745 --- /dev/null +++ b/data/subsets/tasks_all/models_randomforest/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RandomForest, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTElasticNetMarkovBlanket(R)ReliefFF-TestLassomRMRGainRatioSBESFSLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_randomforest/budgets_3/pareto.html b/data/subsets/tasks_all/models_randomforest/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..68e0cb8e062227173c59f15e3ca2521090cad792 --- /dev/null +++ b/data/subsets/tasks_all/models_randomforest/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RandomForest, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-Test(R)ReliefFElasticNetLassoGainRatioMarkovBlanketmRMRSBELOCOSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_randomforest/budgets_4/pareto.html b/data/subsets/tasks_all/models_randomforest/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..ae1b1958df8131292dfbcb0f15c6a01b6861d480 --- /dev/null +++ b/data/subsets/tasks_all/models_randomforest/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RandomForest, budget 5
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestMI(R)ReliefFMarkovBlanketElasticNetmRMRCARTGainRatioSBELassoLOCOSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_randomforest/budgets_all/pareto.html b/data/subsets/tasks_all/models_randomforest/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..df63d001ae3c273d2802095998934e1f688d1368 --- /dev/null +++ b/data/subsets/tasks_all/models_randomforest/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RandomForest, all budgets
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-TestMarkovBlanketElasticNetLassomRMRGainRatio(R)ReliefFSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_realmlp/budgets_0/elo_explorer.html b/data/subsets/tasks_all/models_realmlp/budgets_0/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..b65918ccec0cdff5ec0eb238f11136e457c6cc2c --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_0/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 1
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandomLOCOSBE(R)ReliefFGainRatioLassoSFSmRMRElasticNetF-TestMICARTMarkovBlanketRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_realmlp/budgets_0/leaderboard_table.html b/data/subsets/tasks_all/models_realmlp/budgets_0/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..e3106ed0a73335e6f7e050b77dd0f35af3ad2159 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_0/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 1
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1335+52 / −486.3421.40.6280.8050.20.0
🥈2MarkovBlanketBackward searchsearch1324+45 / −456.3522.70.4180.4650.20.0
🥉3CARTTreenon-search1311+56 / −496.3823.80.4840.7310.10.0
4MIInformation theorynon-search1287+43 / −427.0525.10.6450.8980.50.0
5F-TestCorrelationnon-search1285+45 / −437.0426.20.7250.9520.00.0
6ElasticNetRegularizationnon-search1282+51 / −466.9027.20.5090.9157.80.0
7mRMRInformation theorynon-search1268+44 / −447.5125.90.5280.7790.40.0
8SFSForward searchsearch1257+51 / −537.2125.60.3680.750783.229.1
9LassoRegularizationnon-search1245+46 / −477.4627.80.4210.8902.60.0
10GainRatioInformation theorynon-search1235+46 / −458.1828.80.6790.9420.20.0
11(R)ReliefFDistancenon-search1203+52 / −518.5429.70.6420.87317.40.0
12SBEBackward searchsearch1189+52 / −568.7031.20.1950.6213,600.050.2
13LOCOLeave-one-outsearch1162+51 / −498.5731.50.2990.665239.00.0
14RandomBaseline1000+52 / −6411.3243.0-0.0010.4930.00.0
15LaplacianScoreDistancenon-search929+73 / −9512.4447.30.5880.8470.20.0
diff --git a/data/subsets/tasks_all/models_realmlp/budgets_0/pareto.csv b/data/subsets/tasks_all/models_realmlp/budgets_0/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..a78b0cde3de00da57f1004342975b653872f189d --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_0/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,30.2736,54.5672,71,True +F-Test,28.2978,1.1348,71,True +MI,28.0454,58.9324,71,False +CART,27.1975,17.4527,71,False +MarkovBlanket,26.8472,25.5132,71,False +mRMR,25.3112,21.5307,71,False +SFS,23.5444,16245.5898,71,False +ElasticNet,23.2841,2857.3197,71,False +GainRatio,22.8779,16.6326,71,False +Lasso,22.0792,2935.5029,71,False +(R)ReliefF,21.8611,25267.5895,71,False +LOCO,14.7235,6215.5377,71,False +SBE,13.6605,33786.2929,71,False +Random,0.0,0.0372,71,True +LaplacianScore,-17.3077,4.7242,71,False diff --git a/data/subsets/tasks_all/models_realmlp/budgets_0/pareto.html b/data/subsets/tasks_all/models_realmlp/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..818df95a1689687fa1a45f877b506227bb8aa5ac --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 1
Hover or tap a method for its values.
10-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestMICARTMarkovBlanketmRMRSFSElasticNetGainRatioLasso(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_realmlp/budgets_0/website_leaderboard.csv b/data/subsets/tasks_all/models_realmlp/budgets_0/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..cfeef926d83da2d2fea8b6d68dd0ce004ba6dc76 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_0/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1335.0,1287.0,1387.0,6.34,61.8,21.4,,0.628,0.805,0.2,0.0,71,213 +2,MarkovBlanket,Search,Backward search,1324.0,1279.0,1369.0,6.35,61.8,22.7,,0.418,0.465,0.2,0.0,71,213 +3,CART,Non-search,Tree,1311.0,1262.0,1367.0,6.38,61.6,23.8,,0.484,0.731,0.1,0.0,71,213 +4,MI,Non-search,Information theory,1287.0,1245.0,1330.0,7.05,56.8,25.1,,0.645,0.898,0.5,0.0,71,213 +5,F-Test,Non-search,Correlation,1285.0,1242.0,1330.0,7.04,56.9,26.2,,0.725,0.952,0.0,0.0,71,213 +6,ElasticNet,Non-search,Regularization,1282.0,1236.0,1333.0,6.9,57.9,27.2,,0.509,0.915,7.8,0.0,71,213 +7,mRMR,Non-search,Information theory,1268.0,1224.0,1312.0,7.51,53.5,25.9,,0.528,0.779,0.4,0.0,71,213 +8,SFS,Search,Forward search,1257.0,1204.0,1308.0,7.21,55.6,25.6,,0.368,0.75,783.2,29.1,71,213 +9,Lasso,Non-search,Regularization,1245.0,1198.0,1291.0,7.46,53.8,27.8,,0.421,0.89,2.6,0.0,71,213 +10,GainRatio,Non-search,Information theory,1235.0,1190.0,1281.0,8.18,48.7,28.8,,0.679,0.942,0.2,0.0,71,213 +11,(R)ReliefF,Non-search,Distance,1203.0,1152.0,1255.0,8.54,46.1,29.7,,0.642,0.873,17.4,0.0,71,213 +12,SBE,Search,Backward search,1189.0,1133.0,1241.0,8.7,45.0,31.2,,0.195,0.621,3600.0,50.2,71,213 +13,LOCO,Search,Leave-one-out,1162.0,1113.0,1213.0,8.57,45.9,31.5,,0.299,0.665,239.0,0.0,71,213 +14,Random,Baseline,Baseline,1000.0,936.0,1052.0,11.32,26.3,43.0,,-0.001,0.493,0.0,0.0,71,213 +15,LaplacianScore,Non-search,Distance,929.0,834.0,1002.0,12.44,18.3,47.3,,0.588,0.847,0.2,0.0,71,213 diff --git a/data/subsets/tasks_all/models_realmlp/budgets_1/elo_explorer.html b/data/subsets/tasks_all/models_realmlp/budgets_1/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..a86bd41d9549a3156b5b0cdd53914cd6b876f933 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_1/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 2
Hover or tap a method for its values.
900100011001200130014001500EloRandomLaplacianScoreSBELOCOmRMRSFSGainRatio(R)ReliefFMIF-TestMarkovBlanketLassoElasticNetCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_realmlp/budgets_1/leaderboard_table.html b/data/subsets/tasks_all/models_realmlp/budgets_1/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..b8be090dce836e3f6e49dfc9fb1aef2757ee6e29 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_1/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 2
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1349+51 / −456.3920.40.6530.7410.20.0
🥈2CARTTreenon-search1339+49 / −456.3521.40.4560.6380.10.0
🥉3ElasticNetRegularizationnon-search1335+51 / −486.6323.60.5190.8737.80.0
4LassoRegularizationnon-search1307+51 / −536.9724.90.4220.8252.60.0
5MarkovBlanketBackward searchsearch1293+46 / −407.1024.60.3960.4670.20.0
6F-TestCorrelationnon-search1288+44 / −477.3125.40.7340.9320.00.0
7MIInformation theorynon-search1287+45 / −437.4025.50.6730.8630.50.0
8(R)ReliefFDistancenon-search1269+54 / −497.8025.00.6470.80817.20.0
9GainRatioInformation theorynon-search1260+43 / −467.8826.80.6810.9150.20.0
10SFSForward searchsearch1253+46 / −487.8926.70.2780.6641,478.343.2
11mRMRInformation theorynon-search1236+49 / −488.0426.70.5780.7120.50.0
12LOCOLeave-one-outsearch1226+50 / −498.2928.00.2200.613237.00.0
13SBEBackward searchsearch1221+59 / −578.4230.10.1750.5782,842.748.8
14RandomBaseline1000+49 / −5711.5140.90.0010.4990.00.0
15LaplacianScoreDistancenon-search1000+64 / −8212.0243.70.6440.8500.20.0
diff --git a/data/subsets/tasks_all/models_realmlp/budgets_1/pareto.csv b/data/subsets/tasks_all/models_realmlp/budgets_1/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..92433271a1c6fa682bb5b221c1aaefa21610a0cb --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_1/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,30.1766,54.7341,71,True +CART,28.0413,17.5071,71,True +MI,26.9218,58.507,71,False +F-Test,26.3139,0.9308,71,True +ElasticNet,25.1369,2940.3587,71,False +GainRatio,24.5692,16.8409,71,False +(R)ReliefF,24.073,25262.9971,71,False +MarkovBlanket,23.9101,25.3425,71,False +Lasso,23.433,2918.7999,71,False +mRMR,23.4022,29.4245,71,False +SFS,21.74,23487.2347,71,False +LOCO,16.8299,6261.0683,71,False +SBE,13.3295,31087.6089,71,False +Random,0.0,0.003,71,True +LaplacianScore,-28.1319,4.556,71,False diff --git a/data/subsets/tasks_all/models_realmlp/budgets_1/pareto.html b/data/subsets/tasks_all/models_realmlp/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..736f282653b23bb7de68d9dfb04a4c2815002a53 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIF-TestElasticNetGainRatio(R)ReliefFMarkovBlanketLassomRMRSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_realmlp/budgets_1/website_leaderboard.csv b/data/subsets/tasks_all/models_realmlp/budgets_1/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..90a049b8f882f98390a221e9caae924428e518b9 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_1/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1349.0,1304.0,1400.0,6.39,61.5,20.4,,0.653,0.741,0.2,0.0,71,213 +2,CART,Non-search,Tree,1339.0,1294.0,1388.0,6.35,61.8,21.4,,0.456,0.638,0.1,0.0,71,213 +3,ElasticNet,Non-search,Regularization,1335.0,1287.0,1386.0,6.63,59.8,23.6,,0.519,0.873,7.8,0.0,71,213 +4,Lasso,Non-search,Regularization,1307.0,1254.0,1358.0,6.97,57.4,24.9,,0.422,0.825,2.6,0.0,71,213 +5,MarkovBlanket,Search,Backward search,1293.0,1253.0,1339.0,7.1,56.4,24.6,,0.396,0.467,0.2,0.0,71,213 +6,F-Test,Non-search,Correlation,1288.0,1241.0,1332.0,7.31,54.9,25.4,,0.734,0.932,0.0,0.0,71,213 +7,MI,Non-search,Information theory,1287.0,1244.0,1332.0,7.4,54.3,25.5,,0.673,0.863,0.5,0.0,71,213 +8,(R)ReliefF,Non-search,Distance,1269.0,1220.0,1323.0,7.8,51.5,25.0,,0.647,0.808,17.2,0.0,71,213 +9,GainRatio,Non-search,Information theory,1260.0,1214.0,1303.0,7.88,50.9,26.8,,0.681,0.915,0.2,0.0,71,213 +10,SFS,Search,Forward search,1253.0,1205.0,1299.0,7.89,50.7,26.7,,0.278,0.664,1478.3,43.2,71,213 +11,mRMR,Non-search,Information theory,1236.0,1188.0,1285.0,8.04,49.7,26.7,,0.578,0.712,0.5,0.0,71,213 +12,LOCO,Search,Leave-one-out,1226.0,1177.0,1276.0,8.29,47.9,28.0,,0.22,0.613,237.0,0.0,71,213 +13,SBE,Search,Backward search,1221.0,1164.0,1280.0,8.42,47.0,30.1,,0.175,0.578,2842.7,48.8,71,213 +14,Random,Baseline,Baseline,1000.0,943.0,1049.0,11.51,24.9,40.9,,0.001,0.499,0.0,0.0,71,213 +15,LaplacianScore,Non-search,Distance,1000.0,918.0,1064.0,12.02,21.3,43.7,,0.644,0.85,0.2,0.0,71,213 diff --git a/data/subsets/tasks_all/models_realmlp/budgets_2/elo_explorer.html b/data/subsets/tasks_all/models_realmlp/budgets_2/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..74e6451de723445e6aa5d170b8f8e1bc67b15d27 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_2/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 3
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandommRMRLOCOSFSSBEGainRatio(R)ReliefFF-TestLassoMIMarkovBlanketElasticNetCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_realmlp/budgets_2/leaderboard_table.html b/data/subsets/tasks_all/models_realmlp/budgets_2/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..65c0544bec6ee11bf0cf0a5f9bd96dd71be7db5b --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_2/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 3
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1336+57 / −485.9816.50.6760.6910.20.0
🥈2CARTTreenon-search1298+53 / −456.4920.30.4440.5720.10.0
🥉3ElasticNetRegularizationnon-search1296+54 / −516.5221.50.4810.7777.80.0
4MarkovBlanketBackward searchsearch1264+42 / −447.3323.40.4130.4470.20.0
5MIInformation theorynon-search1262+46 / −466.9521.50.6830.8150.50.0
6LassoRegularizationnon-search1250+50 / −567.1922.00.3890.7302.50.0
7F-TestCorrelationnon-search1223+49 / −497.7223.50.7060.8790.00.0
8(R)ReliefFDistancenon-search1218+52 / −527.8122.90.6730.75717.50.0
9GainRatioInformation theorynon-search1208+47 / −468.0925.60.6700.8560.20.0
10SBEBackward searchsearch1198+52 / −488.1228.00.1440.5562,524.347.9
11SFSForward searchsearch1180+46 / −508.0324.50.2390.6272,618.545.5
12LOCOLeave-one-outsearch1177+49 / −498.3525.90.1510.575241.67.5
13mRMRInformation theorynon-search1166+42 / −488.3224.60.6140.6600.60.0
14RandomBaseline1000+47 / −5510.8135.0-0.0010.5130.00.0
15LaplacianScoreDistancenon-search893+65 / −8412.3041.50.6600.8270.20.0
diff --git a/data/subsets/tasks_all/models_realmlp/budgets_2/pareto.csv b/data/subsets/tasks_all/models_realmlp/budgets_2/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..cf81c03ee6e8e61aefd524a275bd161cfb0284c6 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_2/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,24.6618,55.1181,71,True +CART,21.1491,17.4784,71,True +MarkovBlanket,19.2737,25.4412,71,False +MI,18.7737,59.0125,71,False +GainRatio,17.4805,16.5286,71,True +(R)ReliefF,17.1723,25260.0962,71,False +ElasticNet,16.5467,2899.1848,71,False +F-Test,16.2846,0.9719,71,True +Lasso,14.1035,2972.8973,71,False +mRMR,13.915,46.7518,71,False +SFS,10.9344,29042.103,71,False +SBE,9.1413,28030.9397,71,False +LOCO,9.1022,6424.8431,71,False +Random,0.0,0.0035,71,True +LaplacianScore,-108.4466,5.0671,71,False diff --git a/data/subsets/tasks_all/models_realmlp/budgets_2/pareto.html b/data/subsets/tasks_all/models_realmlp/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..b4d7ef9618a8194763514f692eded04486154d8a --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMarkovBlanketMIGainRatio(R)ReliefFElasticNetF-TestLassomRMRSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_realmlp/budgets_2/website_leaderboard.csv b/data/subsets/tasks_all/models_realmlp/budgets_2/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..5de590b40408a9cee6cbaf1f5c0db3c1cc960632 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_2/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1336.0,1288.0,1393.0,5.98,64.4,16.5,,0.676,0.691,0.2,0.0,71,213 +2,CART,Non-search,Tree,1298.0,1253.0,1351.0,6.49,60.8,20.3,,0.444,0.572,0.1,0.0,71,213 +3,ElasticNet,Non-search,Regularization,1296.0,1245.0,1350.0,6.52,60.6,21.5,,0.481,0.777,7.8,0.0,71,213 +4,MarkovBlanket,Search,Backward search,1264.0,1220.0,1306.0,7.33,54.8,23.4,,0.413,0.447,0.2,0.0,71,213 +5,MI,Non-search,Information theory,1262.0,1216.0,1308.0,6.95,57.5,21.5,,0.683,0.815,0.5,0.0,71,213 +6,Lasso,Non-search,Regularization,1250.0,1194.0,1300.0,7.19,55.8,22.0,,0.389,0.73,2.5,0.0,71,213 +7,F-Test,Non-search,Correlation,1223.0,1174.0,1272.0,7.72,52.0,23.5,,0.706,0.879,0.0,0.0,71,213 +8,(R)ReliefF,Non-search,Distance,1218.0,1166.0,1270.0,7.81,51.4,22.9,,0.673,0.757,17.5,0.0,71,213 +9,GainRatio,Non-search,Information theory,1208.0,1162.0,1255.0,8.09,49.4,25.6,,0.67,0.856,0.2,0.0,71,213 +10,SBE,Search,Backward search,1198.0,1150.0,1250.0,8.12,49.2,28.0,,0.144,0.556,2524.3,47.9,71,213 +11,SFS,Search,Forward search,1180.0,1130.0,1226.0,8.03,49.8,24.5,,0.239,0.627,2618.5,45.5,71,213 +12,LOCO,Search,Leave-one-out,1177.0,1128.0,1226.0,8.35,47.5,25.9,,0.151,0.575,241.6,7.5,71,213 +13,mRMR,Non-search,Information theory,1166.0,1118.0,1208.0,8.32,47.7,24.6,,0.614,0.66,0.6,0.0,71,213 +14,Random,Baseline,Baseline,1000.0,945.0,1047.0,10.81,30.0,35.0,,-0.001,0.513,0.0,0.0,71,213 +15,LaplacianScore,Non-search,Distance,893.0,809.0,958.0,12.3,19.3,41.5,,0.66,0.827,0.2,0.0,71,213 diff --git a/data/subsets/tasks_all/models_realmlp/budgets_3/elo_explorer.html b/data/subsets/tasks_all/models_realmlp/budgets_3/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..bc909590a57c6bfcae74ff6350dde7640094903e --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_3/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 4
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandomGainRatiomRMRSFSLassoSBELOCOF-TestMarkovBlanket(R)ReliefFElasticNetMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_realmlp/budgets_3/leaderboard_table.html b/data/subsets/tasks_all/models_realmlp/budgets_3/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..a2eb85ebb304828398e2ae9bf26062ff0c52efc5 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_3/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 4
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1259+50 / −476.4115.40.6610.6000.20.0
🥈2CARTTreenon-search1253+56 / −476.4716.70.4020.5070.10.0
🥉3MIInformation theorynon-search1229+50 / −466.8817.00.6490.7230.50.0
4ElasticNetRegularizationnon-search1208+48 / −537.1618.90.4190.6657.90.0
5(R)ReliefFDistancenon-search1190+52 / −467.6519.80.6560.66717.40.0
6MarkovBlanketBackward searchsearch1178+41 / −478.0320.00.4340.4730.20.0
7F-TestCorrelationnon-search1176+43 / −437.3918.30.6680.7810.00.0
8SBEBackward searchsearch1162+56 / −497.8322.00.0950.5302,084.644.6
9LOCOLeave-one-outsearch1162+48 / −487.9321.10.1140.544245.313.2
10LassoRegularizationnon-search1160+47 / −497.6520.50.3340.6432.50.0
11SFSForward searchsearch1146+41 / −408.2421.20.2250.5682,685.249.3
12mRMRInformation theorynon-search1143+44 / −488.1020.00.6560.5930.67.0
13GainRatioInformation theorynon-search1134+46 / −438.5420.40.6410.7510.20.0
14RandomBaseline1000+49 / −599.7926.60.0020.4940.00.0
15LaplacianScoreDistancenon-search905+62 / −8011.9135.20.6890.7580.20.0
diff --git a/data/subsets/tasks_all/models_realmlp/budgets_3/pareto.csv b/data/subsets/tasks_all/models_realmlp/budgets_3/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..d5b29ce28e6cd5bcd792cf9294f0955d1124fab4 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_3/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,15.9189,56.0205,70,True +CART,14.7503,17.8368,70,True +MI,12.7053,59.8355,70,False +(R)ReliefF,10.9144,25615.1538,70,False +F-Test,10.9,0.9385,70,True +GainRatio,10.8263,16.8126,70,False +mRMR,10.793,83.768,70,False +MarkovBlanket,10.1043,24.4916,70,False +LOCO,9.8762,6377.4567,70,False +ElasticNet,9.0201,2848.0425,70,False +SBE,7.7488,23659.79,70,False +SFS,7.5706,32319.6092,70,False +Lasso,6.7661,2937.6039,70,False +Random,0.0,0.0033,70,True +LaplacianScore,-42.7443,4.6157,70,False diff --git a/data/subsets/tasks_all/models_realmlp/budgets_3/pareto.html b/data/subsets/tasks_all/models_realmlp/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..6b0055d92be0a7776866e1a0e765f2cd619f4565 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 4
Hover or tap a method for its values.
10-210-11001011021031040246810121416Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMI(R)ReliefFF-TestGainRatiomRMRMarkovBlanketLOCOElasticNetSBESFSLassoRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_realmlp/budgets_3/website_leaderboard.csv b/data/subsets/tasks_all/models_realmlp/budgets_3/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..1da1ac6983a2807fe6eec4cebf810d8b465acaaf --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_3/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1259.0,1212.0,1309.0,6.41,61.4,15.4,,0.661,0.6,0.2,0.0,71,213 +2,CART,Non-search,Tree,1253.0,1206.0,1309.0,6.47,60.9,16.7,,0.402,0.507,0.1,0.0,71,213 +3,MI,Non-search,Information theory,1229.0,1183.0,1279.0,6.88,58.0,17.0,,0.649,0.723,0.5,0.0,71,213 +4,ElasticNet,Non-search,Regularization,1208.0,1155.0,1256.0,7.16,56.0,18.9,,0.419,0.665,7.9,0.0,71,213 +5,(R)ReliefF,Non-search,Distance,1190.0,1144.0,1242.0,7.65,52.5,19.8,,0.656,0.667,17.4,0.0,71,213 +6,MarkovBlanket,Search,Backward search,1178.0,1131.0,1219.0,8.03,49.8,20.0,,0.434,0.473,0.2,0.0,71,213 +7,F-Test,Non-search,Correlation,1176.0,1133.0,1219.0,7.39,54.4,18.3,,0.668,0.781,0.0,0.0,71,213 +8,SBE,Search,Backward search,1162.0,1113.0,1218.0,7.83,51.2,22.0,,0.095,0.53,2084.6,44.6,71,213 +9,LOCO,Search,Leave-one-out,1162.0,1114.0,1210.0,7.93,50.5,21.1,,0.114,0.544,245.3,13.2,71,213 +10,Lasso,Non-search,Regularization,1160.0,1111.0,1207.0,7.65,52.5,20.5,,0.334,0.643,2.5,0.0,71,213 +11,SFS,Search,Forward search,1146.0,1106.0,1187.0,8.24,48.3,21.2,,0.225,0.568,2685.2,49.3,71,213 +12,mRMR,Non-search,Information theory,1143.0,1095.0,1187.0,8.1,49.3,20.0,,0.656,0.593,0.6,7.0,71,213 +13,GainRatio,Non-search,Information theory,1134.0,1091.0,1180.0,8.54,46.1,20.4,,0.641,0.751,0.2,0.0,71,213 +14,Random,Baseline,Baseline,1000.0,941.0,1049.0,9.79,37.2,26.6,,0.002,0.494,0.0,0.0,71,213 +15,LaplacianScore,Non-search,Distance,905.0,825.0,967.0,11.91,22.1,35.2,,0.689,0.758,0.2,0.0,71,213 diff --git a/data/subsets/tasks_all/models_realmlp/budgets_4/elo_explorer.html b/data/subsets/tasks_all/models_realmlp/budgets_4/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..a1e87b60bbee5e7cf8daa920aa3b776621beb5d8 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_4/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 5
Hover or tap a method for its values.
9001000110012001300EloLaplacianScoreRandomGainRatioSFSMarkovBlanketmRMRSBELassoLOCOF-TestElasticNet(R)ReliefFMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_realmlp/budgets_4/leaderboard_table.html b/data/subsets/tasks_all/models_realmlp/budgets_4/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..72c9833caf76407d55533995ad04792ee3645718 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_4/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 5
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1198+55 / −466.4714.90.5970.5070.20.0
🥈2CARTTreenon-search1187+50 / −476.5514.90.3240.4820.10.0
🥉3MIInformation theorynon-search1166+47 / −447.0916.00.5870.6270.50.0
4(R)ReliefFDistancenon-search1149+52 / −477.6216.90.6120.59417.20.0
5ElasticNetRegularizationnon-search1145+49 / −477.3116.60.3080.5689.20.0
6F-TestCorrelationnon-search1129+48 / −527.3916.00.5620.6890.00.0
7LOCOLeave-one-outsearch1110+44 / −457.8518.20.0800.520244.319.7
8LassoRegularizationnon-search1101+47 / −487.5917.40.2460.5632.50.0
9SBEBackward searchsearch1093+46 / −498.1418.60.0590.517888.942.7
10mRMRInformation theorynon-search1084+45 / −508.0417.00.6510.5300.79.9
11MarkovBlanketBackward searchsearch1067+43 / −488.3018.70.4040.4630.20.0
12SFSForward searchsearch1055+44 / −488.7319.70.1760.5252,931.349.3
13GainRatioInformation theorynon-search1027+39 / −469.0419.40.5500.6590.20.0
14RandomBaseline1000+46 / −549.6522.50.0020.5000.00.0
15LaplacianScoreDistancenon-search967+62 / −7110.2325.50.6780.7010.20.0
diff --git a/data/subsets/tasks_all/models_realmlp/budgets_4/pareto.csv b/data/subsets/tasks_all/models_realmlp/budgets_4/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..19daef28eb0031687a05ebcc9ab7de3c5f9672ce --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_4/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,8.0215,54.8184,71,True +CART,7.967,17.7037,71,True +F-Test,7.0893,0.9276,71,True +ElasticNet,6.1547,2821.3905,71,False +(R)ReliefF,6.0796,25242.1614,71,False +mRMR,5.3521,148.6236,71,False +LOCO,5.1839,6168.6284,71,False +SBE,4.8921,15575.2314,71,False +MarkovBlanket,4.8083,24.0853,71,False +MI,3.9736,58.7689,71,False +Lasso,3.0129,2932.8855,71,False +SFS,1.1871,33586.2368,71,False +GainRatio,0.9621,16.6051,71,False +Random,0.0,0.0039,71,True +LaplacianScore,-54816896872894.945,4.8468,71,False diff --git a/data/subsets/tasks_all/models_realmlp/budgets_4/pareto.html b/data/subsets/tasks_all/models_realmlp/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..55de5d41e000678d8f29a5e5770226298f8694e0 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RealMLP, budget 5
Hover or tap a method for its values.
10-210-1100101102103104012345678Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTF-TestElasticNet(R)ReliefFmRMRLOCOSBEMarkovBlanketMILassoSFSGainRatioRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_realmlp/budgets_4/website_leaderboard.csv b/data/subsets/tasks_all/models_realmlp/budgets_4/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..5c5344daa5412496b2335682587d4865be6ce604 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_4/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1198.0,1152.0,1253.0,6.47,60.9,14.9,,0.597,0.507,0.2,0.0,71,213 +2,CART,Non-search,Tree,1187.0,1140.0,1237.0,6.55,60.4,14.9,,0.324,0.482,0.1,0.0,71,213 +3,MI,Non-search,Information theory,1166.0,1122.0,1213.0,7.09,56.5,16.0,,0.587,0.627,0.5,0.0,71,213 +4,(R)ReliefF,Non-search,Distance,1149.0,1102.0,1201.0,7.62,52.7,16.9,,0.612,0.594,17.2,0.0,71,213 +5,ElasticNet,Non-search,Regularization,1145.0,1098.0,1194.0,7.31,55.0,16.6,,0.308,0.568,9.2,0.0,71,213 +6,F-Test,Non-search,Correlation,1129.0,1077.0,1177.0,7.39,54.4,16.0,,0.562,0.689,0.0,0.0,71,213 +7,LOCO,Search,Leave-one-out,1110.0,1065.0,1154.0,7.85,51.1,18.2,,0.08,0.52,244.3,19.7,71,213 +8,Lasso,Non-search,Regularization,1101.0,1053.0,1148.0,7.59,52.9,17.4,,0.246,0.563,2.5,0.0,71,213 +9,SBE,Search,Backward search,1093.0,1044.0,1139.0,8.14,49.0,18.6,,0.059,0.517,888.9,42.7,71,213 +10,mRMR,Non-search,Information theory,1084.0,1034.0,1129.0,8.04,49.7,17.0,,0.651,0.53,0.7,9.9,71,213 +11,MarkovBlanket,Search,Backward search,1067.0,1019.0,1110.0,8.3,47.8,18.7,,0.404,0.463,0.2,0.0,71,213 +12,SFS,Search,Forward search,1055.0,1007.0,1099.0,8.73,44.8,19.7,,0.176,0.525,2931.3,49.3,71,213 +13,GainRatio,Non-search,Information theory,1027.0,981.0,1066.0,9.04,42.6,19.4,,0.55,0.659,0.2,0.0,71,213 +14,Random,Baseline,Baseline,1000.0,946.0,1046.0,9.65,38.2,22.5,,0.002,0.5,0.0,0.0,71,213 +15,LaplacianScore,Non-search,Distance,967.0,896.0,1029.0,10.23,34.1,25.5,,0.678,0.701,0.2,0.0,71,213 diff --git a/data/subsets/tasks_all/models_realmlp/budgets_all/elo_explorer.html b/data/subsets/tasks_all/models_realmlp/budgets_all/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..ae08743e1ab0e93dba51ac4bfbf5012454e3c847 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_all/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, RealMLP, all budgets
Hover or tap a method for its values.
90010001100120013001400EloLaplacianScoreRandomLOCOSBEGainRatioSFSmRMR(R)ReliefFLassoF-TestMarkovBlanketMIElasticNetCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_realmlp/budgets_all/leaderboard_table.html b/data/subsets/tasks_all/models_realmlp/budgets_all/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..a7d551c4733085a69f9e33daacbe37003d847d12 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_all/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, RealMLP, all budgets
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1289+35 / −356.3217.70.6430.6690.20.0
🥈2CARTTreenon-search1271+40 / −336.4519.40.4220.5860.10.0
🥉3ElasticNetRegularizationnon-search1247+35 / −396.9021.60.4470.7607.80.0
4MIInformation theorynon-search1240+33 / −317.0821.00.6470.7850.50.0
5MarkovBlanketBackward searchsearch1218+30 / −277.4221.90.4130.4630.20.0
6F-TestCorrelationnon-search1214+33 / −337.3721.90.6790.8460.00.0
7LassoRegularizationnon-search1206+33 / −377.3722.50.3620.7302.60.0
8(R)ReliefFDistancenon-search1200+39 / −367.8822.80.6460.74017.40.0
9mRMRInformation theorynon-search1174+31 / −338.0022.80.6040.6550.53.4
10SFSForward searchsearch1172+33 / −328.0223.60.2670.6272,080.943.3
11SBEBackward searchsearch1167+41 / −388.2426.00.1320.5602,390.146.8
12GainRatioInformation theorynon-search1167+31 / −318.3524.20.6440.8250.20.0
13LOCOLeave-one-outsearch1162+37 / −338.2024.90.1730.583242.68.1
14RandomBaseline1000+35 / −3610.6233.60.0010.5000.00.0
15LaplacianScoreDistancenon-search942+48 / −6011.7838.60.6520.7970.20.0
diff --git a/data/subsets/tasks_all/models_realmlp/budgets_all/pareto.csv b/data/subsets/tasks_all/models_realmlp/budgets_all/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..dbcfc59b17060b5f4e286bf6ce54e5c96b6ab918 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_all/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,27.0392,54.8943,71,True +CART,24.9132,17.546,71,True +MI,24.2144,58.8456,71,False +F-Test,23.128,0.9784,71,True +MarkovBlanket,21.8261,25.1168,71,False +mRMR,21.5824,66.4533,71,False +ElasticNet,20.9298,2865.3077,71,False +(R)ReliefF,20.6316,25257.4449,71,False +GainRatio,20.3239,16.6431,71,False +Lasso,19.2202,2931.2641,71,False +SFS,18.7021,26845.7831,71,False +LOCO,14.7581,6272.2168,71,False +SBE,12.5247,26362.0022,71,False +Random,0.0,0.0102,71,True +LaplacianScore,-18.397,4.7494,71,False diff --git a/data/subsets/tasks_all/models_realmlp/budgets_all/pareto.html b/data/subsets/tasks_all/models_realmlp/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..b71d2ada1d230e5f20e5801d02dac492b9e9088e --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, RealMLP, all budgets
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIF-TestMarkovBlanketmRMRElasticNet(R)ReliefFGainRatioLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_realmlp/budgets_all/website_leaderboard.csv b/data/subsets/tasks_all/models_realmlp/budgets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..59f5c1081de481f0465e73249bbd1b24d5f507d4 --- /dev/null +++ b/data/subsets/tasks_all/models_realmlp/budgets_all/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1289.0,1254.0,1324.0,6.32,62.0,17.7,,0.643,0.669,0.2,0.0,71,1065 +2,CART,Non-search,Tree,1271.0,1238.0,1311.0,6.45,61.1,19.4,,0.422,0.586,0.1,0.0,71,1065 +3,ElasticNet,Non-search,Regularization,1247.0,1208.0,1282.0,6.9,57.8,21.6,,0.447,0.76,7.8,0.0,71,1065 +4,MI,Non-search,Information theory,1240.0,1209.0,1273.0,7.08,56.6,21.0,,0.647,0.785,0.5,0.0,71,1065 +5,MarkovBlanket,Search,Backward search,1218.0,1191.0,1248.0,7.42,54.1,21.9,,0.413,0.463,0.2,0.0,71,1065 +6,F-Test,Non-search,Correlation,1214.0,1181.0,1247.0,7.37,54.5,21.9,,0.679,0.846,0.0,0.0,71,1065 +7,Lasso,Non-search,Regularization,1206.0,1169.0,1239.0,7.37,54.5,22.5,,0.362,0.73,2.6,0.0,71,1065 +8,(R)ReliefF,Non-search,Distance,1200.0,1164.0,1239.0,7.88,50.8,22.8,,0.646,0.74,17.4,0.0,71,1065 +9,mRMR,Non-search,Information theory,1174.0,1141.0,1205.0,8.0,50.0,22.8,,0.604,0.655,0.5,3.4,71,1065 +10,SFS,Search,Forward search,1172.0,1140.0,1205.0,8.02,49.8,23.6,,0.267,0.627,2080.9,43.3,71,1065 +11,SBE,Search,Backward search,1167.0,1129.0,1208.0,8.24,48.3,26.0,,0.132,0.56,2390.1,46.8,71,1065 +12,GainRatio,Non-search,Information theory,1167.0,1136.0,1198.0,8.35,47.5,24.2,,0.644,0.825,0.2,0.0,71,1065 +13,LOCO,Search,Leave-one-out,1162.0,1129.0,1199.0,8.2,48.6,24.9,,0.173,0.583,242.6,8.1,71,1065 +14,Random,Baseline,Baseline,1000.0,964.0,1035.0,10.62,31.3,33.6,,0.001,0.5,0.0,0.0,71,1065 +15,LaplacianScore,Non-search,Distance,942.0,882.0,990.0,11.78,23.0,38.6,,0.652,0.797,0.2,0.0,71,1065 diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_0/elo_explorer.html b/data/subsets/tasks_all/models_tabiclv2/budgets_0/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..70ab93c1975dfedf53c88ded55aba6eef06345aa --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_0/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 1
Hover or tap a method for its values.
900100011001200130014001500EloLaplacianScoreRandomSBELOCO(R)ReliefFGainRatiomRMRSFSLassoMIElasticNetF-TestCARTMarkovBlanketRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_0/leaderboard_table.html b/data/subsets/tasks_all/models_tabiclv2/budgets_0/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..4c88d8167ae8ff7b2ead6f96fde36e2650997a15 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_0/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 1
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1359+55 / −485.9919.40.6280.8050.20.0
🥈2MarkovBlanketBackward searchsearch1319+54 / −496.4420.50.4180.4650.20.0
🥉3CARTTreenon-search1317+51 / −436.4923.50.4840.7310.10.0
4F-TestCorrelationnon-search1282+45 / −447.2024.30.7250.9520.00.0
5ElasticNetRegularizationnon-search1281+46 / −457.0524.50.5090.9157.80.0
6MIInformation theorynon-search1273+47 / −437.1323.30.6450.8980.50.0
7LassoRegularizationnon-search1250+43 / −447.5226.50.4210.8902.60.0
8SFSForward searchsearch1248+49 / −517.4226.00.3680.750829.828.6
9mRMRInformation theorynon-search1246+45 / −457.7425.00.5280.7790.40.0
10GainRatioInformation theorynon-search1237+48 / −498.2126.60.6790.9420.20.0
11(R)ReliefFDistancenon-search1213+48 / −538.3428.70.6420.87317.40.0
12LOCOLeave-one-outsearch1206+49 / −448.2430.10.2990.665237.00.0
13SBEBackward searchsearch1194+57 / −558.5431.10.1950.6213,240.549.8
14RandomBaseline1000+47 / −5811.3943.4-0.0010.4930.00.0
15LaplacianScoreDistancenon-search966+65 / −8212.2947.40.5880.8470.20.0
diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_0/pareto.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_0/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..4e5c633d9574f3ed1e6411e51d35b6d305b719dc --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_0/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,32.1813,54.6604,71,True +MI,29.9566,58.5868,71,False +F-Test,29.1825,0.9685,71,True +MarkovBlanket,29.1276,25.8011,71,False +CART,28.5797,17.45,71,False +mRMR,27.1434,21.6269,71,False +ElasticNet,25.5029,2895.1726,71,False +GainRatio,24.2515,16.5495,71,False +(R)ReliefF,23.803,25231.0886,71,False +SFS,23.6328,16208.9902,71,False +Lasso,23.3879,2908.0466,71,False +LOCO,16.3464,6213.5283,71,False +SBE,14.3036,33101.807,71,False +Random,0.0,0.0115,71,True +LaplacianScore,-17.1726,4.7319,71,False diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_0/pareto.html b/data/subsets/tasks_all/models_tabiclv2/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..cd31136500b07b7731807a8004a409e624001466 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIF-TestMarkovBlanketCARTmRMRElasticNetGainRatio(R)ReliefFSFSLassoLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_0/website_leaderboard.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_0/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..da6cda558ba483d44c4211f7c0050b801c80fd6f --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_0/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1359.0,1311.0,1414.0,5.99,64.4,19.4,,0.628,0.805,0.2,0.0,71,213 +2,MarkovBlanket,Search,Backward search,1319.0,1270.0,1373.0,6.44,61.1,20.5,,0.418,0.465,0.2,0.0,71,213 +3,CART,Non-search,Tree,1317.0,1274.0,1368.0,6.49,60.8,23.5,,0.484,0.731,0.1,0.0,71,213 +4,F-Test,Non-search,Correlation,1282.0,1238.0,1327.0,7.2,55.7,24.3,,0.725,0.952,0.0,0.0,71,213 +5,ElasticNet,Non-search,Regularization,1281.0,1236.0,1327.0,7.05,56.8,24.5,,0.509,0.915,7.8,0.0,71,213 +6,MI,Non-search,Information theory,1273.0,1230.0,1320.0,7.13,56.2,23.3,,0.645,0.898,0.5,0.0,71,213 +7,Lasso,Non-search,Regularization,1250.0,1206.0,1293.0,7.52,53.4,26.5,,0.421,0.89,2.6,0.0,71,213 +8,SFS,Search,Forward search,1248.0,1197.0,1297.0,7.42,54.1,26.0,,0.368,0.75,829.8,28.6,71,213 +9,mRMR,Non-search,Information theory,1246.0,1201.0,1291.0,7.74,51.9,25.0,,0.528,0.779,0.4,0.0,71,213 +10,GainRatio,Non-search,Information theory,1237.0,1188.0,1285.0,8.21,48.5,26.6,,0.679,0.942,0.2,0.0,71,213 +11,(R)ReliefF,Non-search,Distance,1213.0,1160.0,1261.0,8.34,47.6,28.7,,0.642,0.873,17.4,0.0,71,213 +12,LOCO,Search,Leave-one-out,1206.0,1162.0,1255.0,8.24,48.3,30.1,,0.299,0.665,237.0,0.0,71,213 +13,SBE,Search,Backward search,1194.0,1139.0,1251.0,8.54,46.2,31.1,,0.195,0.621,3240.5,49.8,71,213 +14,Random,Baseline,Baseline,1000.0,942.0,1047.0,11.39,25.8,43.4,,-0.001,0.493,0.0,0.0,71,213 +15,LaplacianScore,Non-search,Distance,966.0,884.0,1031.0,12.29,19.3,47.4,,0.588,0.847,0.2,0.0,71,213 diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_1/elo_explorer.html b/data/subsets/tasks_all/models_tabiclv2/budgets_1/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..def55d4f42c192280dbea262cea2e2a5acaba736 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_1/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 2
Hover or tap a method for its values.
900100011001200130014001500EloRandomLaplacianScoreSBELOCOSFSGainRatio(R)ReliefFmRMRLassoMarkovBlanketF-TestElasticNetMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_1/leaderboard_table.html b/data/subsets/tasks_all/models_tabiclv2/budgets_1/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..ff16e783dcb65ee6dc21a5056f43aa25df567642 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_1/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 2
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1387+52 / −526.0916.80.6530.7410.20.0
🥈2CARTTreenon-search1356+46 / −476.4521.70.4560.6380.10.0
🥉3MIInformation theorynon-search1341+41 / −406.9321.00.6730.8630.50.0
4ElasticNetRegularizationnon-search1324+45 / −456.8523.30.5190.8737.90.0
5MarkovBlanketBackward searchsearch1320+49 / −396.9323.80.3960.4670.20.0
6F-TestCorrelationnon-search1320+47 / −507.3124.80.7340.9320.00.0
7LassoRegularizationnon-search1309+50 / −517.2421.20.4220.8252.50.0
8mRMRInformation theorynon-search1277+49 / −477.8824.20.5780.7120.50.0
9(R)ReliefFDistancenon-search1276+46 / −448.0025.20.6470.80817.40.0
10GainRatioInformation theorynon-search1272+43 / −417.8626.80.6810.9150.20.0
11SFSForward searchsearch1246+50 / −488.0225.90.2780.6641,551.343.2
12LOCOLeave-one-outsearch1244+53 / −508.2628.20.2200.613235.20.0
13SBEBackward searchsearch1231+59 / −578.4329.20.1750.5783,186.648.8
14LaplacianScoreDistancenon-search1014+67 / −8212.0444.50.6440.8500.20.0
15RandomBaseline1000+50 / −6111.7041.00.0010.4990.00.0
diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_1/pareto.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_1/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..fdb8f4b2abe8c6a0a91ccfa5c5bc29d39856b455 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_1/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,31.505,55.1429,71,True +MI,29.7891,58.5821,71,False +CART,29.2997,17.5365,71,True +MarkovBlanket,26.8412,25.4388,71,False +mRMR,26.5707,29.11,71,False +F-Test,26.4949,0.9297,71,True +ElasticNet,25.1703,2876.4429,71,False +GainRatio,24.66,16.4524,71,False +(R)ReliefF,24.17,25295.6416,71,False +Lasso,24.0787,2926.4416,71,False +SFS,22.2697,23701.2276,71,False +LOCO,17.1518,6244.1962,71,False +SBE,15.4018,31048.3548,71,False +Random,0.0,0.0029,71,True +LaplacianScore,-28.206,4.5869,71,False diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_1/pareto.html b/data/subsets/tasks_all/models_tabiclv2/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..303036538efe9bd2a0a28443cf68185778ae2ca7 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTMarkovBlanketmRMRF-TestElasticNetGainRatio(R)ReliefFLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_1/website_leaderboard.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_1/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..22513e2a096f0b5817df4b08d7e24897731970e5 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_1/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1387.0,1335.0,1439.0,6.09,63.6,16.8,,0.653,0.741,0.2,0.0,71,213 +2,CART,Non-search,Tree,1356.0,1309.0,1402.0,6.45,61.1,21.7,,0.456,0.638,0.1,0.0,71,213 +3,MI,Non-search,Information theory,1341.0,1301.0,1382.0,6.93,57.6,21.0,,0.673,0.863,0.5,0.0,71,213 +4,ElasticNet,Non-search,Regularization,1324.0,1279.0,1369.0,6.85,58.2,23.3,,0.519,0.873,7.9,0.0,71,213 +5,MarkovBlanket,Search,Backward search,1320.0,1281.0,1369.0,6.93,57.6,23.8,,0.396,0.467,0.2,0.0,71,213 +6,F-Test,Non-search,Correlation,1320.0,1270.0,1367.0,7.31,55.0,24.8,,0.734,0.932,0.0,0.0,71,213 +7,Lasso,Non-search,Regularization,1309.0,1258.0,1359.0,7.24,55.4,21.2,,0.422,0.825,2.5,0.0,71,213 +8,mRMR,Non-search,Information theory,1277.0,1230.0,1326.0,7.88,50.9,24.2,,0.578,0.712,0.5,0.0,71,213 +9,(R)ReliefF,Non-search,Distance,1276.0,1232.0,1322.0,8.0,50.0,25.2,,0.647,0.808,17.4,0.0,71,213 +10,GainRatio,Non-search,Information theory,1272.0,1231.0,1315.0,7.86,51.0,26.8,,0.681,0.915,0.2,0.0,71,213 +11,SFS,Search,Forward search,1246.0,1198.0,1296.0,8.02,49.9,25.9,,0.278,0.664,1551.3,43.2,71,213 +12,LOCO,Search,Leave-one-out,1244.0,1194.0,1297.0,8.26,48.1,28.2,,0.22,0.613,235.2,0.0,71,213 +13,SBE,Search,Backward search,1231.0,1174.0,1290.0,8.43,46.9,29.2,,0.175,0.578,3186.6,48.8,71,213 +14,LaplacianScore,Non-search,Distance,1014.0,932.0,1081.0,12.04,21.1,44.5,,0.644,0.85,0.2,0.0,71,213 +15,Random,Baseline,Baseline,1000.0,939.0,1050.0,11.7,23.5,41.0,,0.001,0.499,0.0,0.0,71,213 diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_2/elo_explorer.html b/data/subsets/tasks_all/models_tabiclv2/budgets_2/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..3189f764dbe6f4bd0bf974635595044a0331afa5 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_2/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 3
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandomSFSLOCOGainRatiomRMRSBELassoF-Test(R)ReliefFMarkovBlanketElasticNetMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_2/leaderboard_table.html b/data/subsets/tasks_all/models_tabiclv2/budgets_2/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..89d549f6fbbe88e17afc22aa37f923f72e86d78a --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_2/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 3
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1340+46 / −406.0815.80.6760.6910.20.0
🥈2CARTTreenon-search1311+54 / −486.3017.40.4440.5720.10.0
🥉3MIInformation theorynon-search1285+52 / −486.8519.00.6830.8150.50.0
4ElasticNetRegularizationnon-search1256+50 / −566.9619.60.4810.7777.90.0
5MarkovBlanketBackward searchsearch1253+49 / −437.5021.60.4130.4470.20.0
6(R)ReliefFDistancenon-search1237+47 / −487.5320.10.6730.75717.40.0
7F-TestCorrelationnon-search1231+48 / −447.5021.30.7060.8790.00.0
8LassoRegularizationnon-search1224+52 / −577.2920.70.3890.7302.60.0
9SBEBackward searchsearch1220+51 / −517.7723.80.1440.5562,458.046.9
10mRMRInformation theorynon-search1196+45 / −478.3522.00.6140.6600.50.0
11GainRatioInformation theorynon-search1186+43 / −418.4924.20.6700.8560.20.0
12LOCOLeave-one-outsearch1178+54 / −578.1924.80.1510.575234.67.5
13SFSForward searchsearch1169+47 / −478.2724.40.2390.6272,648.445.5
14RandomBaseline1000+48 / −5610.7334.3-0.0010.5130.00.0
15LaplacianScoreDistancenon-search916+66 / −8312.1941.00.6600.8270.20.0
diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_2/pareto.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_2/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..fe3693c39ee1379a9396ef5eede9cb813ca7e2cf --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_2/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,26.0839,54.6867,71,True +CART,22.5729,17.5836,71,True +MI,20.5773,58.9178,71,False +(R)ReliefF,19.89,25263.3245,71,False +MarkovBlanket,18.6104,25.296,71,False +F-Test,18.5312,0.9474,71,True +mRMR,18.136,46.9766,71,False +ElasticNet,17.8488,2761.0945,71,False +GainRatio,16.1056,16.6931,71,False +Lasso,14.8207,2908.2174,71,False +SBE,12.9209,28157.9122,71,False +SFS,12.402,29679.4653,71,False +LOCO,11.1872,6226.075,71,False +Random,0.0,0.0128,71,True +LaplacianScore,-209.6001,4.97,71,False diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_2/pareto.html b/data/subsets/tasks_all/models_tabiclv2/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..5b34c334a3a7c11cb04553d9695f77c2854c7043 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMI(R)ReliefFMarkovBlanketF-TestmRMRElasticNetGainRatioLassoSBESFSLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_2/website_leaderboard.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_2/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..728a36346b829ab2369f27c57dbd0aeb9daa68d4 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_2/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1340.0,1300.0,1386.0,6.08,63.7,15.8,,0.676,0.691,0.2,0.0,71,213 +2,CART,Non-search,Tree,1311.0,1263.0,1365.0,6.3,62.2,17.4,,0.444,0.572,0.1,0.0,71,213 +3,MI,Non-search,Information theory,1285.0,1237.0,1337.0,6.85,58.2,19.0,,0.683,0.815,0.5,0.0,71,213 +4,ElasticNet,Non-search,Regularization,1256.0,1200.0,1306.0,6.96,57.4,19.6,,0.481,0.777,7.9,0.0,71,213 +5,MarkovBlanket,Search,Backward search,1253.0,1210.0,1302.0,7.5,53.6,21.6,,0.413,0.447,0.2,0.0,71,213 +6,(R)ReliefF,Non-search,Distance,1237.0,1189.0,1284.0,7.53,53.3,20.1,,0.673,0.757,17.4,0.0,71,213 +7,F-Test,Non-search,Correlation,1231.0,1187.0,1279.0,7.5,53.6,21.3,,0.706,0.879,0.0,0.0,71,213 +8,Lasso,Non-search,Regularization,1224.0,1167.0,1276.0,7.29,55.0,20.7,,0.389,0.73,2.6,0.0,71,213 +9,SBE,Search,Backward search,1220.0,1169.0,1271.0,7.77,51.7,23.8,,0.144,0.556,2458.0,46.9,71,213 +10,mRMR,Non-search,Information theory,1196.0,1149.0,1241.0,8.35,47.5,22.0,,0.614,0.66,0.5,0.0,71,213 +11,GainRatio,Non-search,Information theory,1186.0,1145.0,1229.0,8.49,46.5,24.2,,0.67,0.856,0.2,0.0,71,213 +12,LOCO,Search,Leave-one-out,1178.0,1121.0,1232.0,8.19,48.6,24.8,,0.151,0.575,234.6,7.5,71,213 +13,SFS,Search,Forward search,1169.0,1122.0,1216.0,8.27,48.1,24.4,,0.239,0.627,2648.4,45.5,71,213 +14,Random,Baseline,Baseline,1000.0,944.0,1048.0,10.73,30.5,34.3,,-0.001,0.513,0.0,0.0,71,213 +15,LaplacianScore,Non-search,Distance,916.0,833.0,982.0,12.19,20.0,41.0,,0.66,0.827,0.2,0.0,71,213 diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_3/elo_explorer.html b/data/subsets/tasks_all/models_tabiclv2/budgets_3/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..20fc6950832ba314bcd7e19cda8231ae1f1390c5 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_3/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 4
Hover or tap a method for its values.
900100011001200130014001500EloLaplacianScoreRandomLOCOSFSmRMRGainRatioLassoElasticNetMarkovBlanket(R)ReliefFSBEF-TestMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_3/leaderboard_table.html b/data/subsets/tasks_all/models_tabiclv2/budgets_3/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..42354d537c414b0ef8c182177b842dcbe9b1a8d9 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_3/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 4
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1350+59 / −505.4310.90.6610.6000.20.0
🥈2CARTTreenon-search1268+43 / −416.5415.60.4020.5070.10.0
🥉3MIInformation theorynon-search1232+52 / −467.0017.60.6490.7230.50.0
4F-TestCorrelationnon-search1222+46 / −417.2015.20.6680.7810.00.0
5SBEBackward searchsearch1203+50 / −577.5819.20.0950.5301,789.344.6
6(R)ReliefFDistancenon-search1199+48 / −507.5216.70.6560.66717.20.0
7MarkovBlanketBackward searchsearch1192+48 / −478.0619.80.4340.4730.20.0
8ElasticNetRegularizationnon-search1185+49 / −507.7518.00.4190.6657.70.0
9LassoRegularizationnon-search1182+51 / −497.6918.20.3340.6432.50.0
10GainRatioInformation theorynon-search1158+40 / −388.3919.70.6410.7510.20.0
11mRMRInformation theorynon-search1151+47 / −498.5517.40.6560.5930.67.0
12SFSForward searchsearch1142+48 / −528.1320.20.2250.5683,266.249.3
13LOCOLeave-one-outsearch1139+49 / −478.0920.80.1140.544241.113.6
14RandomBaseline1000+52 / −6410.1726.30.0020.4940.00.0
15LaplacianScoreDistancenon-search932+67 / −7611.9033.80.6890.7580.20.0
diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_3/pareto.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_3/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..0060287c16032d5f438edb62a9e6ffe9a62256e1 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_3/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,20.1587,55.1453,71,True +CART,14.4246,17.434,71,True +F-Test,11.1647,0.9523,71,True +mRMR,10.9292,85.8167,71,False +(R)ReliefF,9.433,25289.9617,71,False +MarkovBlanket,8.8534,25.0852,71,False +MI,7.6492,58.8803,71,False +GainRatio,7.0524,16.5254,71,False +SBE,6.8028,23102.7346,71,False +LOCO,5.8673,6237.5978,71,False +ElasticNet,5.8292,2769.7249,71,False +SFS,5.7398,32305.4703,71,False +Lasso,5.3886,2922.7829,71,False +Random,0.0,0.0028,71,True +LaplacianScore,-39154926337823.32,4.7054,71,False diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_3/pareto.html b/data/subsets/tasks_all/models_tabiclv2/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..1f51d48146bfa3140614220dc403d2226b3a5158 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTF-TestmRMR(R)ReliefFMarkovBlanketMIGainRatioSBELOCOElasticNetSFSLassoRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_3/website_leaderboard.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_3/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..eab6c337860997bd332b6f27c1ecb84135f5e6f9 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_3/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1350.0,1300.0,1409.0,5.43,68.3,10.9,,0.661,0.6,0.2,0.0,71,213 +2,CART,Non-search,Tree,1268.0,1227.0,1311.0,6.54,60.4,15.6,,0.402,0.507,0.1,0.0,71,213 +3,MI,Non-search,Information theory,1232.0,1186.0,1284.0,7.0,57.1,17.6,,0.649,0.723,0.5,0.0,71,213 +4,F-Test,Non-search,Correlation,1222.0,1181.0,1268.0,7.2,55.7,15.2,,0.668,0.781,0.0,0.0,71,213 +5,SBE,Search,Backward search,1203.0,1146.0,1253.0,7.58,53.0,19.2,,0.095,0.53,1789.3,44.6,71,213 +6,(R)ReliefF,Non-search,Distance,1199.0,1149.0,1247.0,7.52,53.4,16.7,,0.656,0.667,17.2,0.0,71,213 +7,MarkovBlanket,Search,Backward search,1192.0,1145.0,1240.0,8.06,49.6,19.8,,0.434,0.473,0.2,0.0,71,213 +8,ElasticNet,Non-search,Regularization,1185.0,1135.0,1234.0,7.75,51.8,18.0,,0.419,0.665,7.7,0.0,71,213 +9,Lasso,Non-search,Regularization,1182.0,1133.0,1233.0,7.69,52.2,18.2,,0.334,0.643,2.5,0.0,71,213 +10,GainRatio,Non-search,Information theory,1158.0,1120.0,1198.0,8.39,47.2,19.7,,0.641,0.751,0.2,0.0,71,213 +11,mRMR,Non-search,Information theory,1151.0,1102.0,1198.0,8.55,46.1,17.4,,0.656,0.593,0.6,7.0,71,213 +12,SFS,Search,Forward search,1142.0,1090.0,1190.0,8.13,49.1,20.2,,0.225,0.568,3266.2,49.3,71,213 +13,LOCO,Search,Leave-one-out,1139.0,1092.0,1188.0,8.09,49.4,20.8,,0.114,0.544,241.1,13.6,71,213 +14,Random,Baseline,Baseline,1000.0,936.0,1052.0,10.17,34.5,26.3,,0.002,0.494,0.0,0.0,71,213 +15,LaplacianScore,Non-search,Distance,932.0,856.0,999.0,11.9,22.2,33.8,,0.689,0.758,0.2,0.0,71,213 diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_4/elo_explorer.html b/data/subsets/tasks_all/models_tabiclv2/budgets_4/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..861d89f3e7aec0e1d00f6cc3b6faa3d5dbdf8fde --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_4/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 5
Hover or tap a method for its values.
90010001100120013001400EloLaplacianScoreRandomLOCOMarkovBlanketGainRatioSFSmRMRSBELassoMIElasticNetCART(R)ReliefFF-TestRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_4/leaderboard_table.html b/data/subsets/tasks_all/models_tabiclv2/budgets_4/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..16ff4af96eb140d083d992d3bc3c131ca2e103f5 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_4/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 5
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1307+48 / −475.719.20.5970.5070.20.0
🥈2F-TestCorrelationnon-search1204+52 / −497.4010.60.5620.6890.00.0
🥉3(R)ReliefFDistancenon-search1202+52 / −467.4112.60.6120.59417.20.0
4CARTTreenon-search1200+42 / −447.0012.40.3240.4820.10.0
5ElasticNetRegularizationnon-search1185+48 / −457.2811.60.3080.5687.90.0
6MIInformation theorynon-search1167+49 / −517.5913.60.5870.6270.50.0
7LassoRegularizationnon-search1146+51 / −487.7413.00.2460.5632.50.0
8SBEBackward searchsearch1137+46 / −538.0514.80.0590.517866.842.0
9SFSForward searchsearch1126+48 / −497.8614.40.1760.5252,896.548.6
10mRMRInformation theorynon-search1126+47 / −458.3013.50.6510.5300.79.9
11GainRatioInformation theorynon-search1119+46 / −478.3614.90.5500.6590.20.0
12MarkovBlanketBackward searchsearch1114+51 / −508.4415.10.4040.4630.20.0
13LOCOLeave-one-outsearch1107+44 / −468.3215.00.0800.520229.518.8
14RandomBaseline1000+50 / −519.7218.80.0020.5000.00.0
15LaplacianScoreDistancenon-search970+58 / −6710.7624.80.6780.7010.20.0
diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_4/pareto.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_4/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..9a17dae9de8e7bbd353bcd2d929de51de960baa6 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_4/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,12.3414,54.5499,71,True +F-Test,10.8507,1.4247,71,True +(R)ReliefF,8.4909,25307.8206,71,False +CART,8.1108,17.6245,71,False +ElasticNet,7.343,2832.618,71,False +GainRatio,6.7276,16.5018,71,False +Lasso,6.2095,2911.4884,71,False +LOCO,5.6614,6269.599,71,False +MarkovBlanket,5.2199,24.1427,71,False +mRMR,4.1122,148.5945,71,False +SBE,3.5323,15340.395,71,False +SFS,2.0232,34092.5457,71,False +Random,0.0,0.0341,71,True +MI,-7830985267548.507,58.3305,71,False +LaplacianScore,-93466598354697.61,4.6868,71,False diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_4/pareto.html b/data/subsets/tasks_all/models_tabiclv2/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..5d85f20ae1dbf71c0557dfa5d658672dae0902c5 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, TabICLv2, budget 5
Hover or tap a method for its values.
10-1100101102103104-8e+12-7e+12-6e+12-5e+12-4e+12-3e+12-2e+12-1e+120Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-Test(R)ReliefFCARTElasticNetGainRatioLassoLOCOMarkovBlanketmRMRSBESFSRandomMIDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_4/website_leaderboard.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_4/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..31711bdb8a45d530e9878a498d3dd1d54c96ba2c --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_4/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1307.0,1260.0,1355.0,5.71,66.3,9.2,,0.597,0.507,0.2,0.0,71,213 +2,F-Test,Non-search,Correlation,1204.0,1155.0,1256.0,7.4,54.3,10.6,,0.562,0.689,0.0,0.0,71,213 +3,(R)ReliefF,Non-search,Distance,1202.0,1156.0,1254.0,7.41,54.2,12.6,,0.612,0.594,17.2,0.0,71,213 +4,CART,Non-search,Tree,1200.0,1156.0,1242.0,7.0,57.2,12.4,,0.324,0.482,0.1,0.0,71,213 +5,ElasticNet,Non-search,Regularization,1185.0,1140.0,1233.0,7.28,55.1,11.6,,0.308,0.568,7.9,0.0,71,213 +6,MI,Non-search,Information theory,1167.0,1116.0,1216.0,7.59,52.9,13.6,,0.587,0.627,0.5,0.0,71,213 +7,Lasso,Non-search,Regularization,1146.0,1098.0,1197.0,7.74,51.8,13.0,,0.246,0.563,2.5,0.0,71,213 +8,SBE,Search,Backward search,1137.0,1084.0,1183.0,8.05,49.7,14.8,,0.059,0.517,866.8,42.0,71,212 +9,SFS,Search,Forward search,1126.0,1077.0,1174.0,7.86,51.0,14.4,,0.176,0.525,2896.5,48.6,71,212 +10,mRMR,Non-search,Information theory,1126.0,1081.0,1173.0,8.3,47.8,13.5,,0.651,0.53,0.7,9.9,71,213 +11,GainRatio,Non-search,Information theory,1119.0,1072.0,1165.0,8.36,47.4,14.9,,0.55,0.659,0.2,0.0,71,213 +12,MarkovBlanket,Search,Backward search,1114.0,1064.0,1165.0,8.44,46.8,15.1,,0.404,0.463,0.2,0.0,71,213 +13,LOCO,Search,Leave-one-out,1107.0,1061.0,1151.0,8.32,47.6,15.0,,0.08,0.52,229.5,18.8,71,213 +14,Random,Baseline,Baseline,1000.0,949.0,1050.0,9.72,37.7,18.8,,0.002,0.5,0.0,0.0,71,213 +15,LaplacianScore,Non-search,Distance,970.0,903.0,1028.0,10.76,30.2,24.8,,0.678,0.701,0.2,0.0,71,213 diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_all/elo_explorer.html b/data/subsets/tasks_all/models_tabiclv2/budgets_all/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..893dd5188d0d400c7b37414899c9b42678a265d9 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_all/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: all tasks, TabICLv2, all budgets
Hover or tap a method for its values.
90010001100120013001400EloLaplacianScoreRandomLOCOSFSGainRatioSBEmRMRLasso(R)ReliefFMarkovBlanketElasticNetF-TestMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_all/leaderboard_table.html b/data/subsets/tasks_all/models_tabiclv2/budgets_all/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..fdff745d68c1bfd758908f429d09ca2ad33422db --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_all/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: all tasks, TabICLv2, all budgets
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1344+33 / −335.8614.40.6430.6690.20.0
🥈2CARTTreenon-search1286+39 / −356.5518.10.4220.5860.10.0
🥉3MIInformation theorynon-search1255+35 / −327.1018.90.6470.7850.50.0
4F-TestCorrelationnon-search1248+34 / −337.3219.30.6790.8460.00.0
5ElasticNetRegularizationnon-search1242+35 / −387.1819.40.4470.7607.90.0
6MarkovBlanketBackward searchsearch1235+36 / −337.4820.20.4130.4630.20.0
7(R)ReliefFDistancenon-search1221+36 / −357.7620.60.6460.74017.40.0
8LassoRegularizationnon-search1218+35 / −377.5019.90.3620.7302.50.0
9mRMRInformation theorynon-search1195+34 / −338.1620.40.6040.6550.53.4
10SBEBackward searchsearch1193+41 / −408.0723.60.1320.5602,164.246.4
11GainRatioInformation theorynon-search1191+33 / −338.2622.40.6440.8250.20.0
12SFSForward searchsearch1183+34 / −387.9422.20.2670.6272,090.243.0
13LOCOLeave-one-outsearch1171+39 / −388.2223.80.1730.583235.68.0
14RandomBaseline1000+34 / −4010.7432.80.0010.5000.00.0
15LaplacianScoreDistancenon-search960+51 / −6211.8438.30.6520.7970.20.0
diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_all/pareto.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_all/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..3646b442d4b041a8e94f9597226025940083c141 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_all/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,29.0337,54.837,71,True +CART,25.9125,17.5257,71,True +MI,25.4563,58.6595,71,False +F-Test,24.197,1.0445,71,True +mRMR,23.5926,66.425,71,False +MarkovBlanket,23.542,25.1528,71,False +ElasticNet,21.7247,2827.0106,71,False +(R)ReliefF,21.6818,25277.5674,71,False +GainRatio,20.8154,16.5445,71,False +Lasso,20.2986,2915.3954,71,False +SFS,19.3388,27197.5468,71,False +LOCO,14.6377,6238.1993,71,False +SBE,13.1672,26150.2443,71,False +Random,0.0,0.0128,71,True +LaplacianScore,-19.6423,4.7362,71,False diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_all/pareto.html b/data/subsets/tasks_all/models_tabiclv2/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..d425d194e62f102ff49a660b85c837372410a4f1 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: all tasks, TabICLv2, all budgets
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIF-TestmRMRMarkovBlanketElasticNet(R)ReliefFGainRatioLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_all/models_tabiclv2/budgets_all/website_leaderboard.csv b/data/subsets/tasks_all/models_tabiclv2/budgets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..868ce6c4433a1eb81e40572ae8be8e2c24d87792 --- /dev/null +++ b/data/subsets/tasks_all/models_tabiclv2/budgets_all/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1344.0,1311.0,1377.0,5.86,65.3,14.4,,0.643,0.669,0.2,0.0,71,1065 +2,CART,Non-search,Tree,1286.0,1251.0,1325.0,6.55,60.3,18.1,,0.422,0.586,0.1,0.0,71,1065 +3,MI,Non-search,Information theory,1255.0,1223.0,1290.0,7.1,56.4,18.9,,0.647,0.785,0.5,0.0,71,1065 +4,F-Test,Non-search,Correlation,1248.0,1215.0,1282.0,7.32,54.8,19.3,,0.679,0.846,0.0,0.0,71,1065 +5,ElasticNet,Non-search,Regularization,1242.0,1204.0,1277.0,7.18,55.9,19.4,,0.447,0.76,7.9,0.0,71,1065 +6,MarkovBlanket,Search,Backward search,1235.0,1202.0,1271.0,7.48,53.7,20.2,,0.413,0.463,0.2,0.0,71,1065 +7,(R)ReliefF,Non-search,Distance,1221.0,1186.0,1257.0,7.76,51.7,20.6,,0.646,0.74,17.4,0.0,71,1065 +8,Lasso,Non-search,Regularization,1218.0,1181.0,1253.0,7.5,53.6,19.9,,0.362,0.73,2.5,0.0,71,1065 +9,mRMR,Non-search,Information theory,1195.0,1162.0,1229.0,8.16,48.8,20.4,,0.604,0.655,0.5,3.4,71,1065 +10,SBE,Search,Backward search,1193.0,1153.0,1234.0,8.07,49.5,23.6,,0.132,0.56,2164.2,46.4,71,1064 +11,GainRatio,Non-search,Information theory,1191.0,1158.0,1224.0,8.26,48.1,22.4,,0.644,0.825,0.2,0.0,71,1065 +12,SFS,Search,Forward search,1183.0,1145.0,1217.0,7.94,50.4,22.2,,0.267,0.627,2090.2,43.0,71,1064 +13,LOCO,Search,Leave-one-out,1171.0,1133.0,1210.0,8.22,48.4,23.8,,0.173,0.583,235.6,8.0,71,1065 +14,Random,Baseline,Baseline,1000.0,960.0,1034.0,10.74,30.4,32.8,,0.001,0.5,0.0,0.0,71,1065 +15,LaplacianScore,Non-search,Distance,960.0,898.0,1011.0,11.84,22.6,38.3,,0.652,0.797,0.2,0.0,71,1065 diff --git a/data/subsets/tasks_binary/models_all/budgets_0/pareto.html b/data/subsets/tasks_binary/models_all/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..345bc0adb1904cce25f3a6cb9fb6bd2f248e4313 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, all models, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalF-TestRFImportanceMICARTmRMRMarkovBlanketElasticNetSFSLassoGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_all/budgets_0/per_model.csv b/data/subsets/tasks_binary/models_all/budgets_0/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..5653f4446662419a92367c3ab628c00355123633 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_0/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1319.0,1297.0,1,1313.0,2,1328.0,1,1321.0,1,1339.0,1 +2,F-Test,Non-search,Correlation,1292.0,1257.0,5,1357.0,1,1274.0,3,1283.0,3,1293.0,3 +3,MarkovBlanket,Search,Backward search,1278.0,1267.0,4,1283.0,8,1251.0,6,1294.0,2,1302.0,2 +4,CART,Non-search,Tree,1276.0,1274.0,3,1292.0,5,1268.0,4,1273.0,4,1278.0,4 +5,MI,Non-search,Information theory,1273.0,1256.0,6,1293.0,4,1287.0,2,1262.0,8,1270.0,5 +6,SFS,Search,Forward search,1270.0,1282.0,2,1291.0,7,1265.0,5,1270.0,6,1246.0,9 +7,ElasticNet,Non-search,Regularization,1267.0,1253.0,7,1305.0,3,1245.0,7,1264.0,7,1269.0,6 +8,mRMR,Non-search,Information theory,1261.0,1239.0,8,1292.0,6,1243.0,8,1270.0,5,1266.0,7 +9,GainRatio,Non-search,Information theory,1244.0,1222.0,10,1265.0,10,1235.0,9,1246.0,9,1258.0,8 +10,Lasso,Non-search,Regularization,1237.0,1226.0,9,1275.0,9,1207.0,11,1236.0,10,1242.0,10 +11,(R)ReliefF,Non-search,Distance,1194.0,1147.0,13,1207.0,11,1212.0,10,1194.0,11,1208.0,11 +12,SBE,Search,Backward search,1182.0,1204.0,11,1169.0,13,1169.0,12,1185.0,12,1188.0,13 +13,LOCO,Search,Leave-one-out,1176.0,1158.0,12,1196.0,12,1154.0,13,1171.0,13,1205.0,12 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,955.0,904.0,15,1000.0,15,980.0,15,922.0,15,962.0,15 diff --git a/data/subsets/tasks_binary/models_all/budgets_0/per_model_table.html b/data/subsets/tasks_binary/models_all/budgets_0/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..399ec7bde7b2bbc711974113bc975c8b1ae94fab --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_0/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, all models, budget 1 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1319🥇1297🥈1313🥇1328🥇1321🥇1339
🥈2F-Test12921257🥇1357🥉1274🥉1283🥉1293
🥉3MarkovBlanket1278126712831251🥈1294🥈1302
4CART1276🥉12741292126812731278
5MI127312561293🥈128712621270
6SFS1270🥈12821291126512701246
7ElasticNet12671253🥉1305124512641269
8mRMR126112391292124312701266
9GainRatio124412221265123512461258
10Lasso123712261275120712361242
11(R)ReliefF119411471207121211941208
12SBE118212041169116911851188
13LOCO117611581196115411711205
14Random100010001000100010001000
15LaplacianScore9559041000980922962
diff --git a/data/subsets/tasks_binary/models_all/budgets_1/pareto.html b/data/subsets/tasks_binary/models_all/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..a0160c72b9c3a3bb7590339097534b52dc342749 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, all models, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIF-TestCARTElasticNetLassomRMRMarkovBlanketGainRatioSFS(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_all/budgets_1/per_model.csv b/data/subsets/tasks_binary/models_all/budgets_1/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..e5b7fd4128eb45aab8a65d011d03fa467d1e04d3 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_1/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,ElasticNet,Non-search,Regularization,1334.0,1338.0,1,1395.0,1,1316.0,2,1338.0,1,1295.0,4 +2,RFImportance,Non-search,Tree,1313.0,1293.0,8,1312.0,4,1321.0,1,1317.0,2,1331.0,1 +3,Lasso,Non-search,Regularization,1309.0,1317.0,3,1378.0,2,1278.0,3,1311.0,3,1272.0,7 +4,F-Test,Non-search,Correlation,1293.0,1284.0,9,1315.0,3,1269.0,5,1301.0,5,1304.0,3 +5,MI,Non-search,Information theory,1287.0,1305.0,5,1303.0,6,1251.0,8,1274.0,6,1311.0,2 +6,CART,Non-search,Tree,1284.0,1304.0,6,1241.0,10,1278.0,4,1308.0,4,1294.0,5 +7,MarkovBlanket,Search,Backward search,1272.0,1308.0,4,1246.0,9,1258.0,6,1264.0,9,1290.0,6 +8,mRMR,Non-search,Information theory,1271.0,1321.0,2,1303.0,5,1230.0,11,1243.0,12,1269.0,8 +9,GainRatio,Non-search,Information theory,1267.0,1296.0,7,1260.0,8,1254.0,7,1264.0,10,1267.0,9 +10,SFS,Search,Forward search,1260.0,1277.0,10,1278.0,7,1231.0,10,1272.0,7,1251.0,11 +11,(R)ReliefF,Non-search,Distance,1238.0,1259.0,11,1181.0,12,1232.0,9,1265.0,8,1260.0,10 +12,SBE,Search,Backward search,1220.0,1251.0,12,1179.0,13,1228.0,12,1220.0,13,1226.0,12 +13,LOCO,Search,Leave-one-out,1212.0,1226.0,13,1184.0,11,1184.0,13,1250.0,11,1219.0,13 +14,LaplacianScore,Non-search,Distance,1003.0,969.0,15,1014.0,14,1020.0,14,1000.0,15,1006.0,14 +15,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,15,1000.0,15,1000.0,14,1000.0,15 diff --git a/data/subsets/tasks_binary/models_all/budgets_1/per_model_table.html b/data/subsets/tasks_binary/models_all/budgets_1/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..c39aea857797aa785a5860b3ca0587c7130f211a --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_1/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, all models, budget 2 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1ElasticNet1334🥇1338🥇1395🥈1316🥇13381295
🥈2RFImportance131312931312🥇1321🥈1317🥇1331
🥉3Lasso1309🥉1317🥈1378🥉1278🥉13111272
4F-Test12931284🥉131512691301🥉1304
5MI12871305130312511274🥈1311
6CART128413041241127813081294
7MarkovBlanket127213081246125812641290
8mRMR1271🥈13211303123012431269
9GainRatio126712961260125412641267
10SFS126012771278123112721251
11(R)ReliefF123812591181123212651260
12SBE122012511179122812201226
13LOCO121212261184118412501219
14LaplacianScore10039691014102010001006
15Random100010001000100010001000
diff --git a/data/subsets/tasks_binary/models_all/budgets_2/pareto.html b/data/subsets/tasks_binary/models_all/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..79d208b0e6fbee816ed84d163bdd829792650c77 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, all models, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTElasticNetMILassoF-Test(R)ReliefFMarkovBlanketmRMRGainRatioSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_all/budgets_2/per_model.csv b/data/subsets/tasks_binary/models_all/budgets_2/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..b90fb4b01e6d7d29cbb0341a364e2395d86a187c --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_2/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,ElasticNet,Non-search,Regularization,1282.0,1249.0,4,1327.0,1,1283.0,3,1309.0,1,1256.0,5 +2,RFImportance,Non-search,Tree,1281.0,1257.0,1,1216.0,4,1346.0,1,1297.0,2,1298.0,1 +3,MI,Non-search,Information theory,1265.0,1257.0,2,1255.0,3,1288.0,2,1243.0,6,1289.0,2 +4,Lasso,Non-search,Regularization,1252.0,1253.0,3,1295.0,2,1248.0,7,1264.0,4,1209.0,9 +5,CART,Non-search,Tree,1244.0,1239.0,5,1152.0,11,1274.0,4,1281.0,3,1283.0,3 +6,MarkovBlanket,Search,Backward search,1234.0,1227.0,7,1164.0,10,1267.0,5,1257.0,5,1264.0,4 +7,F-Test,Non-search,Correlation,1231.0,1215.0,8,1213.0,5,1249.0,6,1228.0,7,1254.0,6 +8,SFS,Search,Forward search,1203.0,1194.0,10,1199.0,6,1205.0,10,1211.0,9,1212.0,8 +9,SBE,Search,Backward search,1203.0,1227.0,6,1107.0,13,1238.0,9,1201.0,10,1247.0,7 +10,(R)ReliefF,Non-search,Distance,1195.0,1167.0,12,1167.0,9,1240.0,8,1200.0,11,1206.0,10 +11,GainRatio,Non-search,Information theory,1195.0,1200.0,9,1173.0,8,1192.0,11,1217.0,8,1198.0,11 +12,mRMR,Non-search,Information theory,1180.0,1192.0,11,1199.0,7,1164.0,12,1169.0,13,1184.0,12 +13,LOCO,Search,Leave-one-out,1165.0,1163.0,13,1145.0,12,1158.0,13,1188.0,12,1176.0,13 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,920.0,904.0,15,934.0,15,950.0,15,892.0,15,914.0,15 diff --git a/data/subsets/tasks_binary/models_all/budgets_2/per_model_table.html b/data/subsets/tasks_binary/models_all/budgets_2/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..e06132d4e3d6ea579c6ec53b6ced4ee44d112d6c --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_2/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, all models, budget 3 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1ElasticNet12821249🥇1327🥉1283🥇13091256
🥈2RFImportance1281🥇12571216🥇1346🥈1297🥇1298
🥉3MI1265🥈1257🥉1255🥈12881243🥈1289
4Lasso1252🥉1253🥈1295124812641209
5CART1244123911521274🥉1281🥉1283
6MarkovBlanket123412271164126712571264
7F-Test123112151213124912281254
8SFS120311941199120512111212
9SBE120312271107123812011247
10(R)ReliefF119511671167124012001206
11GainRatio119512001173119212171198
12mRMR118011921199116411691184
13LOCO116511631145115811881176
14Random100010001000100010001000
15LaplacianScore920904934950892914
diff --git a/data/subsets/tasks_binary/models_all/budgets_3/pareto.html b/data/subsets/tasks_binary/models_all/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..1285c41fa189003558fb07e40bf25a0fd29a8c4d --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, all models, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.5Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTmRMRF-TestElasticNetMILasso(R)ReliefFGainRatioMarkovBlanketSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_all/budgets_3/per_model.csv b/data/subsets/tasks_binary/models_all/budgets_3/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..3f4e7fa80829cc5b69fa2d1774352cfb07c29d49 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_3/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1235.0,1218.0,1,1183.0,3,1278.0,1,1222.0,3,1287.0,1 +2,CART,Non-search,Tree,1195.0,1184.0,3,1102.0,9,1226.0,2,1230.0,1,1242.0,2 +3,ElasticNet,Non-search,Regularization,1194.0,1198.0,2,1202.0,1,1200.0,3,1211.0,4,1168.0,7 +4,MI,Non-search,Information theory,1186.0,1174.0,6,1162.0,4,1182.0,4,1222.0,2,1198.0,5 +5,F-Test,Non-search,Correlation,1180.0,1184.0,4,1157.0,5,1166.0,6,1163.0,7,1237.0,3 +6,Lasso,Non-search,Regularization,1164.0,1182.0,5,1194.0,2,1161.0,8,1140.0,11,1150.0,8 +7,SBE,Search,Backward search,1158.0,1117.0,9,1120.0,8,1181.0,5,1158.0,8,1219.0,4 +8,mRMR,Non-search,Information theory,1144.0,1163.0,7,1144.0,6,1164.0,7,1123.0,13,1135.0,12 +9,MarkovBlanket,Search,Backward search,1129.0,1114.0,10,1040.0,13,1154.0,9,1172.0,5,1170.0,6 +10,(R)ReliefF,Non-search,Distance,1128.0,1136.0,8,1065.0,11,1140.0,11,1167.0,6,1138.0,10 +11,SFS,Search,Forward search,1120.0,1084.0,13,1136.0,7,1091.0,12,1145.0,10,1148.0,9 +12,GainRatio,Non-search,Information theory,1111.0,1093.0,12,1058.0,12,1141.0,10,1134.0,12,1135.0,11 +13,LOCO,Search,Leave-one-out,1108.0,1096.0,11,1098.0,10,1084.0,13,1150.0,9,1114.0,13 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,919.0,876.0,15,949.0,15,930.0,15,915.0,15,914.0,15 diff --git a/data/subsets/tasks_binary/models_all/budgets_3/per_model_table.html b/data/subsets/tasks_binary/models_all/budgets_3/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..c9989bc93e8055cdcb00cdd590612f590231e8b6 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_3/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, all models, budget 4 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1235🥇1218🥉1183🥇1278🥉1222🥇1287
🥈2CART1195🥉11841102🥈1226🥇1230🥈1242
🥉3ElasticNet1194🥈1198🥇1202🥉120012111168
4MI1186117411621182🥈12221198
5F-Test11801184115711661163🥉1237
6Lasso11641182🥈1194116111401150
7SBE115811171120118111581219
8mRMR114411631144116411231135
9MarkovBlanket112911141040115411721170
10(R)ReliefF112811361065114011671138
11SFS112010841136109111451148
12GainRatio111110931058114111341135
13LOCO110810961098108411501114
14Random100010001000100010001000
15LaplacianScore919876949930915914
diff --git a/data/subsets/tasks_binary/models_all/budgets_4/pareto.html b/data/subsets/tasks_binary/models_all/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..b313a700a23319d7cec61e03c000e0f858e7924b --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, all models, budget 5
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestmRMRElasticNet(R)ReliefFCARTLassoMIMarkovBlanketGainRatioLOCOSBESFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_all/budgets_4/per_model.csv b/data/subsets/tasks_binary/models_all/budgets_4/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..e0cabb296c718e41fbefdd4f9dd67e36f1ab6ff9 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_4/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1241.0,1259.0,1,1194.0,1,1281.0,1,1199.0,1,1286.0,1 +2,CART,Non-search,Tree,1173.0,1163.0,3,1141.0,5,1217.0,2,1184.0,2,1167.0,5 +3,F-Test,Non-search,Correlation,1173.0,1154.0,4,1187.0,2,1192.0,3,1133.0,5,1202.0,2 +4,ElasticNet,Non-search,Regularization,1158.0,1168.0,2,1156.0,4,1170.0,5,1130.0,6,1170.0,3 +5,MI,Non-search,Information theory,1148.0,1138.0,8,1138.0,6,1174.0,4,1150.0,4,1144.0,7 +6,(R)ReliefF,Non-search,Distance,1147.0,1142.0,5,1100.0,9,1156.0,8,1174.0,3,1169.0,4 +7,mRMR,Non-search,Information theory,1137.0,1139.0,7,1177.0,3,1162.0,6,1093.0,9,1120.0,9 +8,Lasso,Non-search,Regularization,1116.0,1141.0,6,1129.0,7,1124.0,11,1078.0,11,1115.0,10 +9,SBE,Search,Backward search,1115.0,1103.0,11,1093.0,10,1128.0,10,1104.0,8,1149.0,6 +10,SFS,Search,Forward search,1110.0,1110.0,9,1117.0,8,1103.0,13,1086.0,10,1135.0,8 +11,MarkovBlanket,Search,Backward search,1101.0,1109.0,10,1084.0,11,1156.0,7,1072.0,12,1088.0,13 +12,LOCO,Search,Leave-one-out,1096.0,1078.0,13,1074.0,12,1112.0,12,1119.0,7,1102.0,11 +13,GainRatio,Non-search,Information theory,1080.0,1087.0,12,1073.0,13,1138.0,9,1012.0,13,1089.0,12 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,15,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,945.0,933.0,15,1007.0,14,892.0,15,936.0,15,944.0,15 diff --git a/data/subsets/tasks_binary/models_all/budgets_4/per_model_table.html b/data/subsets/tasks_binary/models_all/budgets_4/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..18fd678b734b36b81d0de16d199667c0ebe4f67d --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_4/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, all models, budget 5 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1241🥇1259🥇1194🥇1281🥇1199🥇1286
🥈2CART1173🥉11631141🥈1217🥈11841167
🥉3F-Test11731154🥈1187🥉11921133🥈1202
4ElasticNet1158🥈1168115611701130🥉1170
5MI114811381138117411501144
6(R)ReliefF1147114211001156🥉11741169
7mRMR11371139🥉1177116210931120
8Lasso111611411129112410781115
9SBE111511031093112811041149
10SFS111011101117110310861135
11MarkovBlanket110111091084115610721088
12LOCO109610781074111211191102
13GainRatio108010871073113810121089
14Random100010001000100010001000
15LaplacianScore9459331007892936944
diff --git a/data/subsets/tasks_binary/models_all/budgets_all/pareto.html b/data/subsets/tasks_binary/models_all/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..cede3bfafd690ca20581010134ea4552e2dd0912 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, all models, all budgets
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTF-TestMIElasticNetmRMRLassoMarkovBlanketGainRatioSFS(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_all/budgets_all/per_model.csv b/data/subsets/tasks_binary/models_all/budgets_all/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..f53004389c11cbfc7ccd83ed9b99e9da00629791 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_all/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1273.0,1259.0,1,1236.0,4,1307.0,1,1265.0,1,1303.0,1 +2,ElasticNet,Non-search,Regularization,1242.0,1236.0,2,1268.0,1,1240.0,3,1244.0,3,1228.0,5 +3,CART,Non-search,Tree,1230.0,1227.0,3,1179.0,8,1249.0,2,1249.0,2,1248.0,3 +4,F-Test,Non-search,Correlation,1229.0,1214.0,6,1238.0,3,1227.0,5,1216.0,5,1254.0,2 +5,MI,Non-search,Information theory,1227.0,1221.0,4,1223.0,5,1233.0,4,1224.0,4,1238.0,4 +6,Lasso,Non-search,Regularization,1211.0,1218.0,5,1246.0,2,1201.0,7,1200.0,7,1194.0,9 +7,MarkovBlanket,Search,Backward search,1198.0,1199.0,8,1157.0,10,1214.0,6,1206.0,6,1218.0,6 +8,mRMR,Non-search,Information theory,1195.0,1206.0,7,1216.0,6,1190.0,9,1174.0,10,1191.0,11 +9,SFS,Search,Forward search,1188.0,1184.0,9,1198.0,7,1176.0,12,1191.0,9,1194.0,8 +10,(R)ReliefF,Non-search,Distance,1176.0,1165.0,12,1138.0,11,1193.0,8,1194.0,8,1192.0,10 +11,GainRatio,Non-search,Information theory,1175.0,1174.0,11,1159.0,9,1189.0,10,1169.0,12,1185.0,12 +12,SBE,Search,Backward search,1172.0,1175.0,10,1128.0,13,1186.0,11,1168.0,13,1202.0,7 +13,LOCO,Search,Leave-one-out,1148.0,1139.0,13,1134.0,12,1136.0,13,1170.0,11,1160.0,13 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,948.0,917.0,15,981.0,15,953.0,15,935.0,15,948.0,15 diff --git a/data/subsets/tasks_binary/models_all/budgets_all/per_model_table.html b/data/subsets/tasks_binary/models_all/budgets_all/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..bdb93b2e2cf9db9717dcd31e0a471c0c056b2e93 --- /dev/null +++ b/data/subsets/tasks_binary/models_all/budgets_all/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, all models, all budgets · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1273🥇12591236🥇1307🥇1265🥇1303
🥈2ElasticNet1242🥈1236🥇1268🥉1240🥉12441228
🥉3CART1230🥉12271179🥈1249🥈1249🥉1248
4F-Test12291214🥉123812271216🥈1254
5MI122712211223123312241238
6Lasso12111218🥈1246120112001194
7MarkovBlanket119811991157121412061218
8mRMR119512061216119011741191
9SFS118811841198117611911194
10(R)ReliefF117611651138119311941192
11GainRatio117511741159118911691185
12SBE117211751128118611681202
13LOCO114811391134113611701160
14Random100010001000100010001000
15LaplacianScore948917981953935948
diff --git a/data/subsets/tasks_binary/models_lightgbm/budgets_0/pareto.html b/data/subsets/tasks_binary/models_lightgbm/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..a72a2c9c05847aa511453217b98293d42eab1877 --- /dev/null +++ b/data/subsets/tasks_binary/models_lightgbm/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LightGBM, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIF-TestCARTmRMRSFSMarkovBlanketElasticNetLassoGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_lightgbm/budgets_1/pareto.html b/data/subsets/tasks_binary/models_lightgbm/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..cdbe2e798bb3a4477f8668ac3d3807d0ae212b96 --- /dev/null +++ b/data/subsets/tasks_binary/models_lightgbm/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LightGBM, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalMImRMRRFImportanceCARTLassoElasticNetF-TestGainRatioMarkovBlanketSFS(R)ReliefFSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_lightgbm/budgets_2/pareto.html b/data/subsets/tasks_binary/models_lightgbm/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..40527dc747a0cf1a1f28ef1b295c17ed36235b84 --- /dev/null +++ b/data/subsets/tasks_binary/models_lightgbm/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LightGBM, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTElasticNetLassoMIMarkovBlanketmRMRGainRatioF-Test(R)ReliefFSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_lightgbm/budgets_3/pareto.html b/data/subsets/tasks_binary/models_lightgbm/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..b9e7c83368ba81c4d99e4eda22b008f7b09fff37 --- /dev/null +++ b/data/subsets/tasks_binary/models_lightgbm/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LightGBM, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.5Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTF-TestLassomRMRElasticNet(R)ReliefFMIGainRatioMarkovBlanketSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_lightgbm/budgets_4/pareto.html b/data/subsets/tasks_binary/models_lightgbm/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..4ad516af5363fd920c90d77d4df4b47a0815dd27 --- /dev/null +++ b/data/subsets/tasks_binary/models_lightgbm/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LightGBM, budget 5
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestmRMRCARTLassoMarkovBlanket(R)ReliefFElasticNetSBESFSGainRatioMILOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_lightgbm/budgets_all/pareto.html b/data/subsets/tasks_binary/models_lightgbm/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..c0f1236236b606e2d6c898ad974eef1f64327e42 --- /dev/null +++ b/data/subsets/tasks_binary/models_lightgbm/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LightGBM, all budgets
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMImRMRF-TestElasticNetLassoMarkovBlanketGainRatioSFS(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_linearmodel/budgets_0/pareto.html b/data/subsets/tasks_binary/models_linearmodel/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..4c47dfa8612daf1ac558d8e11c0143fdffedc4fc --- /dev/null +++ b/data/subsets/tasks_binary/models_linearmodel/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LinearModel, budget 1
Hover or tap a method for its values.
10-210-110010110210310405101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalF-TestMIRFImportancemRMRElasticNetCARTLassoSFSMarkovBlanketGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_linearmodel/budgets_1/pareto.html b/data/subsets/tasks_binary/models_linearmodel/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..b68ef5a63e5585be292d78a144c9c21ee559a156 --- /dev/null +++ b/data/subsets/tasks_binary/models_linearmodel/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LinearModel, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalLassoF-TestElasticNetRFImportanceMImRMRCARTGainRatioSFSMarkovBlanket(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_linearmodel/budgets_2/pareto.html b/data/subsets/tasks_binary/models_linearmodel/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..460d69458647597f13f32272aee2697c32d42d5f --- /dev/null +++ b/data/subsets/tasks_binary/models_linearmodel/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LinearModel, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceElasticNetLassoMIF-TestmRMR(R)ReliefFCARTSFSGainRatioMarkovBlanketLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_linearmodel/budgets_3/pareto.html b/data/subsets/tasks_binary/models_linearmodel/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..725e9153b7633f1d64830b71bc640b1dbb013e61 --- /dev/null +++ b/data/subsets/tasks_binary/models_linearmodel/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LinearModel, budget 4
Hover or tap a method for its values.
10-210-11001011021031040246810121416Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceElasticNetLassomRMRMICARTF-TestSFS(R)ReliefFGainRatioLOCOMarkovBlanketSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_linearmodel/budgets_4/pareto.html b/data/subsets/tasks_binary/models_linearmodel/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..b0791db8c352226b50dda92f77c1711ae91b5faa --- /dev/null +++ b/data/subsets/tasks_binary/models_linearmodel/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LinearModel, budget 5
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportancemRMRElasticNetF-TestCARTMarkovBlanketGainRatioLassoMI(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_linearmodel/budgets_all/pareto.html b/data/subsets/tasks_binary/models_linearmodel/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..76db0dd6fdcd4ed35ba877e28964f4d9f4103350 --- /dev/null +++ b/data/subsets/tasks_binary/models_linearmodel/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, LinearModel, all budgets
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestMIElasticNetLassomRMRCARTSFSMarkovBlanketGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_randomforest/budgets_0/pareto.html b/data/subsets/tasks_binary/models_randomforest/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..6f3621a1df702de82851033130d5912f32de8e33 --- /dev/null +++ b/data/subsets/tasks_binary/models_randomforest/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RandomForest, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalMIRFImportanceF-TestCARTMarkovBlanketElasticNetmRMRSFSLassoGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_randomforest/budgets_1/pareto.html b/data/subsets/tasks_binary/models_randomforest/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..91846f9ea541c71a3688241b3836324d191db032 --- /dev/null +++ b/data/subsets/tasks_binary/models_randomforest/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RandomForest, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIF-TestElasticNetLassoMarkovBlanket(R)ReliefFmRMRGainRatioSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_randomforest/budgets_2/pareto.html b/data/subsets/tasks_binary/models_randomforest/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..78f9b7f7f987b202ebfbba7077edf476cb8407da --- /dev/null +++ b/data/subsets/tasks_binary/models_randomforest/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RandomForest, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIElasticNetCARTMarkovBlanketF-TestLasso(R)ReliefFGainRatiomRMRSBESFSLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_randomforest/budgets_3/pareto.html b/data/subsets/tasks_binary/models_randomforest/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..ea248169e5c26fd7649d896064fd2c5982449ade --- /dev/null +++ b/data/subsets/tasks_binary/models_randomforest/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RandomForest, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTF-TestMIGainRatio(R)ReliefFElasticNetMarkovBlanketLassomRMRSBELOCORandomSFSDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_randomforest/budgets_4/pareto.html b/data/subsets/tasks_binary/models_randomforest/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..c4dc23e756ced91534ffb38700f0e4b8efd6950d --- /dev/null +++ b/data/subsets/tasks_binary/models_randomforest/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RandomForest, budget 5
Hover or tap a method for its values.
10-210-110010110210310402468101214Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestMIMarkovBlanket(R)ReliefFmRMRElasticNetCARTGainRatioSBELOCOLassoSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_randomforest/budgets_all/pareto.html b/data/subsets/tasks_binary/models_randomforest/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..e96d8613205ea5b4bce817594adc9a55a67a12e9 --- /dev/null +++ b/data/subsets/tasks_binary/models_randomforest/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RandomForest, all budgets
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-TestMarkovBlanketElasticNetLassomRMRGainRatio(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_0/elo_explorer.html b/data/subsets/tasks_binary/models_realmlp/budgets_0/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..a509ba5386338325ed5b722fc075d7d55341587f --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_0/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, RealMLP, budget 1
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandomLOCOSBE(R)ReliefFLassoGainRatioMIElasticNetmRMRSFSCARTF-TestMarkovBlanketRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_0/leaderboard_table.html b/data/subsets/tasks_binary/models_realmlp/budgets_0/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..98d547b10070e10b68ccd43952921c2664a2547b --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_0/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, RealMLP, budget 1
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1321+67 / −636.5824.90.5950.7570.20.0
🥈2MarkovBlanketBackward searchsearch1294+58 / −526.5627.10.4010.4110.30.0
🥉3F-TestCorrelationnon-search1283+56 / −506.7528.60.6990.9360.00.0
4CARTTreenon-search1273+60 / −566.9028.40.4420.6890.10.0
5mRMRInformation theorynon-search1270+56 / −497.2729.20.5070.7530.30.0
6SFSForward searchsearch1270+61 / −686.8927.60.3590.751642.732.0
7ElasticNetRegularizationnon-search1264+61 / −567.0330.90.4860.89513.30.0
8MIInformation theorynon-search1262+53 / −527.4029.10.6010.8570.60.0
9GainRatioInformation theorynon-search1246+50 / −467.7932.10.6770.9560.10.0
10LassoRegularizationnon-search1236+51 / −567.6031.00.3830.8632.40.0
11(R)ReliefFDistancenon-search1194+63 / −648.5633.30.6330.82947.10.0
12SBEBackward searchsearch1185+63 / −608.4335.10.1920.6343,221.549.7
13LOCOLeave-one-outsearch1171+53 / −568.3734.60.2980.648230.30.0
14RandomBaseline1000+67 / −7011.3848.3-0.0010.5110.00.0
15LaplacianScoreDistancenon-search922+89 / −12012.5053.10.5920.8510.20.0
diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_0/pareto.csv b/data/subsets/tasks_binary/models_realmlp/budgets_0/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..9d0ff38648bba501b3553cc5b61bdcad78262e40 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_0/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,33.1063,23.8628,49,True +F-Test,32.8092,1.0216,49,True +MI,31.475,53.0164,49,False +CART,30.887,18.1658,49,False +mRMR,28.9085,9.8375,49,False +MarkovBlanket,28.7133,25.2398,49,False +SFS,27.8122,13667.089,49,False +GainRatio,25.9696,4.7703,49,False +ElasticNet,25.7588,3101.7789,49,False +Lasso,25.5462,2712.1627,49,False +(R)ReliefF,24.715,26745.3338,49,False +LOCO,16.6401,4872.5864,49,False +SBE,14.3357,30981.4088,49,False +Random,0.0,0.0383,49,True +LaplacianScore,-22.4567,4.8595,49,False diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_0/pareto.html b/data/subsets/tasks_binary/models_realmlp/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..ffab3fc060a57a867eaf81982c0f2aaa909db18f --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RealMLP, budget 1
Hover or tap a method for its values.
10-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestMICARTmRMRMarkovBlanketSFSGainRatioElasticNetLasso(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_0/website_leaderboard.csv b/data/subsets/tasks_binary/models_realmlp/budgets_0/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..a36ab039bc7cecebbf29cb328868960ac52b9120 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_0/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1321.0,1258.0,1388.0,6.58,60.1,24.9,,0.595,0.757,0.2,0.0,49,147 +2,MarkovBlanket,Search,Backward search,1294.0,1242.0,1352.0,6.56,60.3,27.1,,0.401,0.411,0.3,0.0,49,147 +3,F-Test,Non-search,Correlation,1283.0,1233.0,1339.0,6.75,58.9,28.6,,0.699,0.936,0.0,0.0,49,147 +4,CART,Non-search,Tree,1273.0,1217.0,1333.0,6.9,57.8,28.4,,0.442,0.689,0.1,0.0,49,147 +5,mRMR,Non-search,Information theory,1270.0,1221.0,1326.0,7.27,55.2,29.2,,0.507,0.753,0.3,0.0,49,147 +6,SFS,Search,Forward search,1270.0,1202.0,1331.0,6.89,57.9,27.6,,0.359,0.751,642.7,32.0,49,147 +7,ElasticNet,Non-search,Regularization,1264.0,1208.0,1325.0,7.03,57.0,30.9,,0.486,0.895,13.3,0.0,49,147 +8,MI,Non-search,Information theory,1262.0,1210.0,1315.0,7.4,54.3,29.1,,0.601,0.857,0.6,0.0,49,147 +9,GainRatio,Non-search,Information theory,1246.0,1200.0,1296.0,7.79,51.5,32.1,,0.677,0.956,0.1,0.0,49,147 +10,Lasso,Non-search,Regularization,1236.0,1180.0,1287.0,7.6,52.9,31.0,,0.383,0.863,2.4,0.0,49,147 +11,(R)ReliefF,Non-search,Distance,1194.0,1130.0,1257.0,8.56,46.0,33.3,,0.633,0.829,47.1,0.0,49,147 +12,SBE,Search,Backward search,1185.0,1125.0,1248.0,8.43,46.9,35.1,,0.192,0.634,3221.5,49.7,49,147 +13,LOCO,Search,Leave-one-out,1171.0,1115.0,1224.0,8.37,47.3,34.6,,0.298,0.648,230.3,0.0,49,147 +14,Random,Baseline,Baseline,1000.0,930.0,1067.0,11.38,25.9,48.3,,-0.001,0.511,0.0,0.0,49,147 +15,LaplacianScore,Non-search,Distance,922.0,802.0,1011.0,12.5,17.9,53.1,,0.592,0.851,0.2,0.0,49,147 diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_1/elo_explorer.html b/data/subsets/tasks_binary/models_realmlp/budgets_1/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..81f34aea0b3d23811cd79f687872d185197d2894 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_1/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, RealMLP, budget 2
Hover or tap a method for its values.
900100011001200130014001500EloRandomLaplacianScoreSBEmRMRLOCOMarkovBlanketGainRatio(R)ReliefFSFSMIF-TestCARTLassoRFImportanceElasticNetstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_1/leaderboard_table.html b/data/subsets/tasks_binary/models_realmlp/budgets_1/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..d7884c463c58133eb71478c172e6799048f57013 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_1/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, RealMLP, budget 2
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1ElasticNetRegularizationnon-search1338+63 / −556.5526.40.4760.84611.80.0
🥈2RFImportanceTreenon-search1317+57 / −566.9324.40.6310.6960.20.0
🥉3LassoRegularizationnon-search1311+59 / −576.8927.90.3600.8002.40.0
4CARTTreenon-search1308+52 / −497.0625.70.4350.6020.10.0
5F-TestCorrelationnon-search1301+49 / −496.9927.70.7100.9160.00.0
6MIInformation theorynon-search1274+49 / −507.7330.10.6320.8150.50.0
7SFSForward searchsearch1272+58 / −577.4929.40.2680.6571,225.444.2
8(R)ReliefFDistancenon-search1265+66 / −637.8328.50.6400.74846.50.0
9MarkovBlanketBackward searchsearch1264+56 / −567.3728.70.3880.4170.20.0
10GainRatioInformation theorynon-search1264+49 / −507.6930.50.6850.9180.10.0
11LOCOLeave-one-outsearch1250+62 / −647.9330.80.2210.589225.40.0
12mRMRInformation theorynon-search1243+56 / −537.9229.80.5600.6820.50.0
13SBEBackward searchsearch1220+66 / −688.2133.90.1780.5432,445.648.3
14RandomBaseline1000+58 / −6911.3045.6-0.0000.4990.00.0
15LaplacianScoreDistancenon-search1000+67 / −9412.0848.90.6550.8530.20.0
diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_1/pareto.csv b/data/subsets/tasks_binary/models_realmlp/budgets_1/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..60cf15ccff17a305075477eb4ba86c1a1f82f557 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_1/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,32.5387,23.8588,49,True +F-Test,30.6961,1.0008,49,True +CART,30.3868,18.2414,49,False +MI,29.1728,52.3347,49,False +ElasticNet,28.5125,3168.1505,49,False +Lasso,27.9613,2685.949,49,False +GainRatio,27.2805,4.7481,49,False +(R)ReliefF,27.239,26736.4739,49,False +mRMR,27.086,17.698,49,False +SFS,25.2143,19824.6662,49,False +MarkovBlanket,24.9679,25.0802,49,False +LOCO,18.0133,4994.8664,49,False +SBE,14.2868,27806.5703,49,False +Random,0.0,0.003,49,True +LaplacianScore,-32.6773,4.6209,49,False diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_1/pareto.html b/data/subsets/tasks_binary/models_realmlp/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..a9a75671750fa09df440a310268f14ef588e7e0d --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RealMLP, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestCARTMIElasticNetLassoGainRatio(R)ReliefFmRMRSFSMarkovBlanketLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_1/website_leaderboard.csv b/data/subsets/tasks_binary/models_realmlp/budgets_1/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..fc500b6fbe6a791c3f90c44b7cffaf2099113b04 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_1/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,ElasticNet,Non-search,Regularization,1338.0,1283.0,1401.0,6.55,60.3,26.4,,0.476,0.846,11.8,0.0,49,147 +2,RFImportance,Non-search,Tree,1317.0,1261.0,1374.0,6.93,57.6,24.4,,0.631,0.696,0.2,0.0,49,147 +3,Lasso,Non-search,Regularization,1311.0,1254.0,1370.0,6.89,57.9,27.9,,0.36,0.8,2.4,0.0,49,147 +4,CART,Non-search,Tree,1308.0,1259.0,1360.0,7.06,56.7,25.7,,0.435,0.602,0.1,0.0,49,147 +5,F-Test,Non-search,Correlation,1301.0,1252.0,1350.0,6.99,57.2,27.7,,0.71,0.916,0.0,0.0,49,147 +6,MI,Non-search,Information theory,1274.0,1224.0,1323.0,7.73,51.9,30.1,,0.632,0.815,0.5,0.0,49,147 +7,SFS,Search,Forward search,1272.0,1215.0,1330.0,7.49,53.6,29.4,,0.268,0.657,1225.4,44.2,49,147 +8,(R)ReliefF,Non-search,Distance,1265.0,1202.0,1331.0,7.83,51.2,28.5,,0.64,0.748,46.5,0.0,49,147 +9,MarkovBlanket,Search,Backward search,1264.0,1208.0,1320.0,7.37,54.5,28.7,,0.388,0.417,0.2,0.0,49,147 +10,GainRatio,Non-search,Information theory,1264.0,1214.0,1313.0,7.69,52.2,30.5,,0.685,0.918,0.1,0.0,49,147 +11,LOCO,Search,Leave-one-out,1250.0,1186.0,1312.0,7.93,50.5,30.8,,0.221,0.589,225.4,0.0,49,147 +12,mRMR,Non-search,Information theory,1243.0,1190.0,1299.0,7.92,50.6,29.8,,0.56,0.682,0.5,0.0,49,147 +13,SBE,Search,Backward search,1220.0,1152.0,1286.0,8.21,48.5,33.9,,0.178,0.543,2445.6,48.3,49,147 +14,Random,Baseline,Baseline,1000.0,931.0,1058.0,11.3,26.4,45.6,,-0.0,0.499,0.0,0.0,49,147 +15,LaplacianScore,Non-search,Distance,1000.0,906.0,1067.0,12.08,20.8,48.9,,0.655,0.853,0.2,0.0,49,147 diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_2/elo_explorer.html b/data/subsets/tasks_binary/models_realmlp/budgets_2/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..94ecbd6f75a51a5dcf3a59da1091c5cbe1622c6f --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_2/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, RealMLP, budget 3
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandommRMRLOCO(R)ReliefFSBESFSGainRatioF-TestMIMarkovBlanketLassoCARTRFImportanceElasticNetstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_2/leaderboard_table.html b/data/subsets/tasks_binary/models_realmlp/budgets_2/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..5ae838c43792da905bf14e8c0293281f2aa52c17 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_2/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, RealMLP, budget 3
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1ElasticNetRegularizationnon-search1309+62 / −566.2323.40.4420.73811.70.0
🥈2RFImportanceTreenon-search1297+65 / −576.6519.60.6680.6570.20.0
🥉3CARTTreenon-search1281+57 / −516.8823.00.4350.5380.10.0
4LassoRegularizationnon-search1264+62 / −547.0323.00.3350.7022.40.0
5MarkovBlanketBackward searchsearch1257+58 / −537.3026.20.4010.3980.30.0
6MIInformation theorynon-search1243+57 / −527.0925.00.6300.7740.50.0
7F-TestCorrelationnon-search1228+55 / −547.7326.90.6830.8640.00.0
8GainRatioInformation theorynon-search1217+58 / −538.0428.80.6730.8650.10.0
9SFSForward searchsearch1211+55 / −607.5926.80.2140.6211,985.144.9
10SBEBackward searchsearch1201+63 / −637.8732.20.1430.5572,118.547.6
11(R)ReliefFDistancenon-search1200+58 / −658.1426.30.6580.70646.80.0
12LOCOLeave-one-outsearch1188+59 / −638.1828.60.1520.545236.18.2
13mRMRInformation theorynon-search1169+55 / −518.2827.20.6130.6330.60.0
14RandomBaseline1000+62 / −6310.7238.7-0.0000.5180.00.0
15LaplacianScoreDistancenon-search892+85 / −12412.2646.00.6560.8240.20.0
diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_2/pareto.csv b/data/subsets/tasks_binary/models_realmlp/budgets_2/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..5ef9d1af9d8065073333e1d954436b57428d669b --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_2/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,26.4137,23.9737,49,True +CART,23.6563,18.2061,49,True +MarkovBlanket,21.2935,25.341,49,False +ElasticNet,19.8821,3135.0997,49,False +Lasso,19.1517,2759.9265,49,False +MI,18.8155,52.8079,49,False +GainRatio,18.7809,4.7596,49,True +(R)ReliefF,18.6047,26745.3531,49,False +F-Test,17.5208,1.0659,49,True +mRMR,14.6111,36.739,49,False +SFS,11.6467,24836.5516,49,False +LOCO,10.3883,5218.3797,49,False +SBE,9.3953,25049.413,49,False +Random,0.0,0.0034,49,True +LaplacianScore,-146.6687,4.8098,49,False diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_2/pareto.html b/data/subsets/tasks_binary/models_realmlp/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..6df9f690d19fda38f7b1e47ca12bb555f2fc90d3 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RealMLP, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMarkovBlanketElasticNetLassoMIGainRatio(R)ReliefFF-TestmRMRSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_2/website_leaderboard.csv b/data/subsets/tasks_binary/models_realmlp/budgets_2/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..dacbba6dac884d65a14f0e5a1bfe5a911ec90266 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_2/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,ElasticNet,Non-search,Regularization,1309.0,1253.0,1371.0,6.23,62.6,23.4,,0.442,0.738,11.7,0.0,49,147 +2,RFImportance,Non-search,Tree,1297.0,1240.0,1362.0,6.65,59.6,19.6,,0.668,0.657,0.2,0.0,49,147 +3,CART,Non-search,Tree,1281.0,1230.0,1338.0,6.88,58.0,23.0,,0.435,0.538,0.1,0.0,49,147 +4,Lasso,Non-search,Regularization,1264.0,1210.0,1326.0,7.03,56.9,23.0,,0.335,0.702,2.4,0.0,49,147 +5,MarkovBlanket,Search,Backward search,1257.0,1204.0,1315.0,7.3,55.0,26.2,,0.401,0.398,0.3,0.0,49,147 +6,MI,Non-search,Information theory,1243.0,1191.0,1300.0,7.09,56.5,25.0,,0.63,0.774,0.5,0.0,49,147 +7,F-Test,Non-search,Correlation,1228.0,1174.0,1283.0,7.73,51.9,26.9,,0.683,0.864,0.0,0.0,49,147 +8,GainRatio,Non-search,Information theory,1217.0,1164.0,1275.0,8.04,49.7,28.8,,0.673,0.865,0.1,0.0,49,147 +9,SFS,Search,Forward search,1211.0,1151.0,1266.0,7.59,52.9,26.8,,0.214,0.621,1985.1,44.9,49,147 +10,SBE,Search,Backward search,1201.0,1138.0,1264.0,7.87,50.9,32.2,,0.143,0.557,2118.5,47.6,49,147 +11,(R)ReliefF,Non-search,Distance,1200.0,1135.0,1258.0,8.14,49.0,26.3,,0.658,0.706,46.8,0.0,49,147 +12,LOCO,Search,Leave-one-out,1188.0,1125.0,1247.0,8.18,48.7,28.6,,0.152,0.545,236.1,8.2,49,147 +13,mRMR,Non-search,Information theory,1169.0,1118.0,1224.0,8.28,48.0,27.2,,0.613,0.633,0.6,0.0,49,147 +14,Random,Baseline,Baseline,1000.0,937.0,1062.0,10.72,30.6,38.7,,-0.0,0.518,0.0,0.0,49,147 +15,LaplacianScore,Non-search,Distance,892.0,768.0,977.0,12.26,19.6,46.0,,0.656,0.824,0.2,0.0,49,147 diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_3/elo_explorer.html b/data/subsets/tasks_binary/models_realmlp/budgets_3/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..839c1c72fa8dbeba109b862cef525385f411a56b --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_3/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, RealMLP, budget 4
Hover or tap a method for its values.
8009001000110012001300EloLaplacianScoreRandommRMRGainRatioLassoSFSLOCOSBEF-Test(R)ReliefFMarkovBlanketElasticNetMIRFImportanceCARTstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_3/leaderboard_table.html b/data/subsets/tasks_binary/models_realmlp/budgets_3/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..c43f415af559b0646e1ef93648a1c40869aa6fcb --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_3/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, RealMLP, budget 4
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1CARTTreenon-search1230+60 / −526.6618.20.3920.4820.10.0
🥈2MIInformation theorynon-search1222+56 / −556.9019.70.6080.6950.50.0
🥉3RFImportanceTreenon-search1222+53 / −527.0318.00.6360.5760.20.0
4ElasticNetRegularizationnon-search1211+69 / −677.1320.60.3730.62411.80.0
5MarkovBlanketBackward searchsearch1172+60 / −527.9822.70.4190.4440.20.0
6(R)ReliefFDistancenon-search1167+61 / −637.8922.00.6410.62945.80.0
7F-TestCorrelationnon-search1163+54 / −547.3219.80.6570.7730.00.0
8SBEBackward searchsearch1158+70 / −687.7224.50.0940.5231,874.945.6
9LOCOLeave-one-outsearch1150+65 / −628.1123.40.1080.520245.312.9
10SFSForward searchsearch1145+51 / −468.0023.30.2170.5622,120.748.3
11LassoRegularizationnon-search1140+58 / −587.7722.00.2850.6152.40.0
12GainRatioInformation theorynon-search1134+50 / −498.3622.60.6320.7700.10.0
13mRMRInformation theorynon-search1123+49 / −548.1721.70.6700.5780.78.2
14RandomBaseline1000+58 / −609.3628.10.0020.4900.00.0
15LaplacianScoreDistancenon-search915+72 / −9611.6038.70.6830.7700.20.0
diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_3/pareto.csv b/data/subsets/tasks_binary/models_realmlp/budgets_3/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..b5a4692085d7f40ef778d7e21accf541429e8d5c --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_3/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,16.0218,24.4236,48,True +CART,14.3904,18.5664,48,True +MI,11.0665,53.5466,48,False +GainRatio,10.8446,4.805,48,True +(R)ReliefF,10.4835,27279.8351,48,False +F-Test,10.2336,1.0191,48,True +mRMR,9.9437,76.3408,48,False +LOCO,9.104,5271.0355,48,False +MarkovBlanket,8.9906,24.056,48,False +ElasticNet,8.2303,3053.3428,48,False +SBE,7.0785,21577.3375,48,False +SFS,5.9778,28678.642,48,False +Lasso,5.6297,2723.7023,48,False +Random,0.0,0.0035,48,True +LaplacianScore,-57.4517,4.6516,48,False diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_3/pareto.html b/data/subsets/tasks_binary/models_realmlp/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..0820a9645898540cf581b3b1c73cca9c11718302 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RealMLP, budget 4
Hover or tap a method for its values.
10-210-11001011021031040246810121416Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIGainRatio(R)ReliefFF-TestmRMRLOCOMarkovBlanketElasticNetSBESFSLassoRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_3/website_leaderboard.csv b/data/subsets/tasks_binary/models_realmlp/budgets_3/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..20b1f7edd4ae7e7d8a0eb2a21d4c6f9042362d2d --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_3/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,CART,Non-search,Tree,1230.0,1178.0,1290.0,6.66,59.6,18.2,,0.392,0.482,0.1,0.0,49,147 +2,MI,Non-search,Information theory,1222.0,1167.0,1278.0,6.9,57.8,19.7,,0.608,0.695,0.5,0.0,49,147 +3,RFImportance,Non-search,Tree,1222.0,1170.0,1275.0,7.03,57.0,18.0,,0.636,0.576,0.2,0.0,49,147 +4,ElasticNet,Non-search,Regularization,1211.0,1144.0,1280.0,7.13,56.2,20.6,,0.373,0.624,11.8,0.0,49,147 +5,MarkovBlanket,Search,Backward search,1172.0,1120.0,1232.0,7.98,50.1,22.7,,0.419,0.444,0.2,0.0,49,147 +6,(R)ReliefF,Non-search,Distance,1167.0,1104.0,1228.0,7.89,50.8,22.0,,0.641,0.629,45.8,0.0,49,147 +7,F-Test,Non-search,Correlation,1163.0,1109.0,1217.0,7.32,54.8,19.8,,0.657,0.773,0.0,0.0,49,147 +8,SBE,Search,Backward search,1158.0,1090.0,1228.0,7.72,52.0,24.5,,0.094,0.523,1874.9,45.6,49,147 +9,LOCO,Search,Leave-one-out,1150.0,1088.0,1215.0,8.11,49.2,23.4,,0.108,0.52,245.3,12.9,49,147 +10,SFS,Search,Forward search,1145.0,1099.0,1196.0,8.0,50.0,23.3,,0.217,0.562,2120.7,48.3,49,147 +11,Lasso,Non-search,Regularization,1140.0,1082.0,1198.0,7.77,51.6,22.0,,0.285,0.615,2.4,0.0,49,147 +12,GainRatio,Non-search,Information theory,1134.0,1085.0,1184.0,8.36,47.4,22.6,,0.632,0.77,0.1,0.0,49,147 +13,mRMR,Non-search,Information theory,1123.0,1069.0,1172.0,8.17,48.8,21.7,,0.67,0.578,0.7,8.2,49,147 +14,Random,Baseline,Baseline,1000.0,940.0,1058.0,9.36,40.3,28.1,,0.002,0.49,0.0,0.0,49,147 +15,LaplacianScore,Non-search,Distance,915.0,819.0,987.0,11.6,24.3,38.7,,0.683,0.77,0.2,0.0,49,147 diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_4/elo_explorer.html b/data/subsets/tasks_binary/models_realmlp/budgets_4/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..cfa98115033639a10407b7ea755a197a95e98156 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_4/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, RealMLP, budget 5
Hover or tap a method for its values.
9001000110012001300EloLaplacianScoreRandomGainRatioMarkovBlanketLassoSFSmRMRSBELOCOElasticNetF-TestMI(R)ReliefFCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_4/leaderboard_table.html b/data/subsets/tasks_binary/models_realmlp/budgets_4/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..964198cb981e920528140ab1c17bc3d132bf1e6d --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_4/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, RealMLP, budget 5
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1199+62 / −536.5016.90.5670.5000.20.0
🥈2CARTTreenon-search1184+61 / −556.5916.90.2790.4750.10.0
🥉3(R)ReliefFDistancenon-search1174+69 / −627.5318.30.5930.57246.70.0
4MIInformation theorynon-search1150+54 / −527.3219.00.5430.6110.50.0
5F-TestCorrelationnon-search1133+57 / −627.3718.30.5490.6840.00.0
6ElasticNetRegularizationnon-search1130+62 / −647.4919.20.2490.54112.40.0
7LOCOLeave-one-outsearch1119+53 / −517.7720.60.0760.513229.420.4
8SBEBackward searchsearch1104+54 / −547.8920.60.0550.513861.843.5
9mRMRInformation theorynon-search1093+59 / −587.9719.30.6860.5100.812.2
10SFSForward searchsearch1086+58 / −578.2921.80.1630.5252,402.147.6
11LassoRegularizationnon-search1078+57 / −537.9619.50.1870.5442.40.0
12MarkovBlanketBackward searchsearch1072+51 / −588.1521.60.4000.4450.20.0
13GainRatioInformation theorynon-search1012+54 / −569.1522.40.5440.6740.10.0
14RandomBaseline1000+63 / −619.4125.60.0020.5010.00.0
15LaplacianScoreDistancenon-search936+70 / −8610.6030.20.6830.7230.20.0
diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_4/pareto.csv b/data/subsets/tasks_binary/models_realmlp/budgets_4/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..3612c2174e130fa4d4a456d659498b98e00804b1 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_4/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +CART,9.9944,18.5205,49,True +RFImportance,8.6624,24.0944,49,False +(R)ReliefF,8.3483,26713.8846,49,False +F-Test,7.941,0.9999,49,True +LOCO,7.3242,5015.5525,49,False +mRMR,5.9506,143.6313,49,False +SBE,5.9141,13989.4737,49,False +ElasticNet,5.8762,2994.4139,49,False +MarkovBlanket,4.4986,24.0072,49,False +Lasso,3.3184,2724.5024,49,False +MI,2.0858,52.2757,49,False +SFS,1.454,29394.7682,49,False +Random,0.0,0.0031,49,True +GainRatio,-0.666,4.7403,49,False +LaplacianScore,-79428564856642.61,4.8285,49,False diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_4/pareto.html b/data/subsets/tasks_binary/models_realmlp/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..aabb144337fd3d01ca822660a9ed6e0dc5720955 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RealMLP, budget 5
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalCARTRFImportance(R)ReliefFF-TestLOCOmRMRSBEElasticNetMarkovBlanketLassoMISFSRandomGainRatioDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_4/website_leaderboard.csv b/data/subsets/tasks_binary/models_realmlp/budgets_4/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..07e5411c40c2fa8c935dfa83c1d42a9c95468eac --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_4/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1199.0,1146.0,1261.0,6.5,60.7,16.9,,0.567,0.5,0.2,0.0,49,147 +2,CART,Non-search,Tree,1184.0,1129.0,1245.0,6.59,60.1,16.9,,0.279,0.475,0.1,0.0,49,147 +3,(R)ReliefF,Non-search,Distance,1174.0,1112.0,1243.0,7.53,53.3,18.3,,0.593,0.572,46.7,0.0,49,147 +4,MI,Non-search,Information theory,1150.0,1098.0,1204.0,7.32,54.8,19.0,,0.543,0.611,0.5,0.0,49,147 +5,F-Test,Non-search,Correlation,1133.0,1071.0,1190.0,7.37,54.5,18.3,,0.549,0.684,0.0,0.0,49,147 +6,ElasticNet,Non-search,Regularization,1130.0,1066.0,1192.0,7.49,53.6,19.2,,0.249,0.541,12.4,0.0,49,147 +7,LOCO,Search,Leave-one-out,1119.0,1068.0,1172.0,7.77,51.6,20.6,,0.076,0.513,229.4,20.4,49,147 +8,SBE,Search,Backward search,1104.0,1050.0,1158.0,7.89,50.8,20.6,,0.055,0.513,861.8,43.5,49,147 +9,mRMR,Non-search,Information theory,1093.0,1035.0,1152.0,7.97,50.2,19.3,,0.686,0.51,0.8,12.2,49,147 +10,SFS,Search,Forward search,1086.0,1029.0,1144.0,8.29,47.9,21.8,,0.163,0.525,2402.1,47.6,49,147 +11,Lasso,Non-search,Regularization,1078.0,1025.0,1135.0,7.96,50.3,19.5,,0.187,0.544,2.4,0.0,49,147 +12,MarkovBlanket,Search,Backward search,1072.0,1014.0,1123.0,8.15,48.9,21.6,,0.4,0.445,0.2,0.0,49,147 +13,GainRatio,Non-search,Information theory,1012.0,956.0,1066.0,9.15,41.8,22.4,,0.544,0.674,0.1,0.0,49,147 +14,Random,Baseline,Baseline,1000.0,939.0,1063.0,9.41,39.9,25.6,,0.002,0.501,0.0,0.0,49,147 +15,LaplacianScore,Non-search,Distance,936.0,850.0,1006.0,10.6,31.4,30.2,,0.683,0.723,0.2,0.0,49,147 diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_all/elo_explorer.html b/data/subsets/tasks_binary/models_realmlp/budgets_all/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..77a882c67cd556be01a2df5e0a937f9959ba6649 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_all/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, RealMLP, all budgets
Hover or tap a method for its values.
90010001100120013001400EloLaplacianScoreRandomSBEGainRatioLOCOmRMRSFS(R)ReliefFLassoMarkovBlanketF-TestMIElasticNetCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_all/leaderboard_table.html b/data/subsets/tasks_binary/models_realmlp/budgets_all/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..2d4dc143b63fd980dc30a2b66048025c77604fcb --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_all/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, RealMLP, all budgets
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1265+42 / −376.7420.80.6190.6370.20.0
🥈2CARTTreenon-search1249+42 / −376.8222.40.3970.5570.10.0
🥉3ElasticNetRegularizationnon-search1244+46 / −466.8924.10.4050.72911.90.0
4MIInformation theorynon-search1224+37 / −347.2924.60.6030.7500.50.0
5F-TestCorrelationnon-search1216+35 / −387.2324.30.6590.8350.00.0
6MarkovBlanketBackward searchsearch1206+39 / −387.4725.30.4020.4230.20.0
7LassoRegularizationnon-search1200+42 / −407.4524.70.3100.7052.40.0
8(R)ReliefFDistancenon-search1194+47 / −467.9925.70.6330.69746.70.0
9SFSForward searchsearch1191+42 / −427.6525.80.2540.6231,745.043.4
10mRMRInformation theorynon-search1174+36 / −357.9225.40.6040.6310.64.1
11LOCOLeave-one-outsearch1170+47 / −458.0727.60.1710.563230.38.3
12GainRatioInformation theorynon-search1169+35 / −358.2127.30.6420.8360.10.0
13SBEBackward searchsearch1168+50 / −488.0229.30.1320.5542,277.446.9
14RandomBaseline1000+39 / −4010.4337.30.0010.5040.00.0
15LaplacianScoreDistancenon-search935+57 / −7211.8143.40.6540.8040.20.0
diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_all/pareto.csv b/data/subsets/tasks_binary/models_realmlp/budgets_all/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..ecb6d0c3bd8ed17f28a687356d2b25344320e3ec --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_all/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,29.3152,23.9436,49,True +CART,27.7382,18.2649,49,True +F-Test,26.3587,1.0178,49,True +MI,26.1119,52.5819,49,False +mRMR,24.2809,57.5079,49,False +ElasticNet,23.2646,3078.1976,49,False +MarkovBlanket,23.0674,24.9525,49,False +(R)ReliefF,22.9765,26732.8308,49,False +Lasso,22.4142,2710.133,49,False +GainRatio,22.2994,4.7545,49,False +SFS,21.3396,23164.2579,49,False +LOCO,16.183,5053.9471,49,False +SBE,13.3032,23793.7507,49,False +Random,0.0,0.0102,49,True +LaplacianScore,-23.4657,4.7357,49,False diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_all/pareto.html b/data/subsets/tasks_binary/models_realmlp/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..6d0cbe1c99ce538403cff6fc37e931d4c8e01015 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, RealMLP, all budgets
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTF-TestMImRMRElasticNetMarkovBlanket(R)ReliefFLassoGainRatioSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_realmlp/budgets_all/website_leaderboard.csv b/data/subsets/tasks_binary/models_realmlp/budgets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..dfcc5fc2e10516db4c3760a64a990da93076bde8 --- /dev/null +++ b/data/subsets/tasks_binary/models_realmlp/budgets_all/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1265.0,1228.0,1307.0,6.74,59.0,20.8,,0.619,0.637,0.2,0.0,49,735 +2,CART,Non-search,Tree,1249.0,1212.0,1291.0,6.82,58.4,22.4,,0.397,0.557,0.1,0.0,49,735 +3,ElasticNet,Non-search,Regularization,1244.0,1198.0,1290.0,6.89,57.9,24.1,,0.405,0.729,11.9,0.0,49,735 +4,MI,Non-search,Information theory,1224.0,1190.0,1261.0,7.29,55.1,24.6,,0.603,0.75,0.5,0.0,49,735 +5,F-Test,Non-search,Correlation,1216.0,1178.0,1251.0,7.23,55.5,24.3,,0.659,0.835,0.0,0.0,49,735 +6,MarkovBlanket,Search,Backward search,1206.0,1168.0,1245.0,7.47,53.8,25.3,,0.402,0.423,0.2,0.0,49,735 +7,Lasso,Non-search,Regularization,1200.0,1160.0,1242.0,7.45,53.9,24.7,,0.31,0.705,2.4,0.0,49,735 +8,(R)ReliefF,Non-search,Distance,1194.0,1148.0,1241.0,7.99,50.1,25.7,,0.633,0.697,46.7,0.0,49,735 +9,SFS,Search,Forward search,1191.0,1149.0,1233.0,7.65,52.5,25.8,,0.254,0.623,1745.0,43.4,49,735 +10,mRMR,Non-search,Information theory,1174.0,1139.0,1210.0,7.92,50.6,25.4,,0.604,0.631,0.6,4.1,49,735 +11,LOCO,Search,Leave-one-out,1170.0,1125.0,1217.0,8.07,49.5,27.6,,0.171,0.563,230.3,8.3,49,735 +12,GainRatio,Non-search,Information theory,1169.0,1134.0,1204.0,8.21,48.5,27.3,,0.642,0.836,0.1,0.0,49,735 +13,SBE,Search,Backward search,1168.0,1120.0,1218.0,8.02,49.8,29.3,,0.132,0.554,2277.4,46.9,49,735 +14,Random,Baseline,Baseline,1000.0,960.0,1039.0,10.43,32.6,37.3,,0.001,0.504,0.0,0.0,49,735 +15,LaplacianScore,Non-search,Distance,935.0,863.0,992.0,11.81,22.8,43.4,,0.654,0.804,0.2,0.0,49,735 diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_0/elo_explorer.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_0/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..614ac275c7e339957b387280f4ada526ca508721 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_0/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 1
Hover or tap a method for its values.
900100011001200130014001500EloLaplacianScoreRandomSBELOCO(R)ReliefFLassoSFSGainRatiomRMRElasticNetMICARTF-TestMarkovBlanketRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_0/leaderboard_table.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_0/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..17051290aab42e856e7e69ace09fa9d2b7d2d380 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_0/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 1
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1339+65 / −646.3523.80.5950.7570.20.0
🥈2MarkovBlanketBackward searchsearch1302+61 / −616.6324.70.4010.4110.30.0
🥉3F-TestCorrelationnon-search1293+51 / −546.9225.80.6990.9360.00.0
4CARTTreenon-search1278+59 / −577.0928.40.4420.6890.10.0
5MIInformation theorynon-search1270+52 / −517.1926.40.6010.8570.50.0
6ElasticNetRegularizationnon-search1269+59 / −577.0528.30.4860.89511.70.0
7mRMRInformation theorynon-search1266+59 / −547.3828.80.5070.7530.30.0
8GainRatioInformation theorynon-search1258+57 / −567.7929.80.6770.9560.10.0
9SFSForward searchsearch1246+61 / −607.3628.80.3590.751761.831.3
10LassoRegularizationnon-search1242+55 / −577.5430.00.3830.8632.40.0
11(R)ReliefFDistancenon-search1208+59 / −598.4332.90.6330.82946.80.0
12LOCOLeave-one-outsearch1205+58 / −578.1233.60.2980.648215.10.0
13SBEBackward searchsearch1188+62 / −678.3735.40.1920.6343,188.349.0
14RandomBaseline1000+57 / −6711.3948.7-0.0010.5110.00.0
15LaplacianScoreDistancenon-search962+76 / −10712.3853.50.5920.8510.20.0
diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_0/pareto.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_0/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..3514f2e36baf289fd9d01553c9fd387cdf3d755d --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_0/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,34.5875,24.0029,49,True +F-Test,33.9174,1.037,49,True +MI,33.2232,52.5452,49,False +MarkovBlanket,31.2112,25.3551,49,False +CART,30.674,18.1371,49,False +mRMR,29.866,10.0836,49,False +GainRatio,27.2387,4.7542,49,False +ElasticNet,27.1192,3089.124,49,False +SFS,26.9712,13845.5746,49,False +(R)ReliefF,25.7653,26692.5809,49,False +Lasso,25.4253,2680.9691,49,False +LOCO,17.9837,4823.9829,49,False +SBE,14.9559,29864.9827,49,False +Random,0.0,0.003,49,True +LaplacianScore,-22.1939,4.8481,49,False diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_0/pareto.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..900a43ff96036f86b611d2d370ab5a6f733602d2 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 1
Hover or tap a method for its values.
10-210-110010110210310405101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestMIMarkovBlanketCARTmRMRGainRatioElasticNetSFS(R)ReliefFLassoLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_0/website_leaderboard.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_0/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..656112dee7726aabe6cc2df8846a9803aa2f6662 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_0/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1339.0,1275.0,1404.0,6.35,61.8,23.8,,0.595,0.757,0.2,0.0,49,147 +2,MarkovBlanket,Search,Backward search,1302.0,1241.0,1363.0,6.63,59.8,24.7,,0.401,0.411,0.3,0.0,49,147 +3,F-Test,Non-search,Correlation,1293.0,1239.0,1344.0,6.92,57.7,25.8,,0.699,0.936,0.0,0.0,49,147 +4,CART,Non-search,Tree,1278.0,1221.0,1337.0,7.09,56.5,28.4,,0.442,0.689,0.1,0.0,49,147 +5,MI,Non-search,Information theory,1270.0,1219.0,1322.0,7.19,55.8,26.4,,0.601,0.857,0.5,0.0,49,147 +6,ElasticNet,Non-search,Regularization,1269.0,1212.0,1328.0,7.05,56.8,28.3,,0.486,0.895,11.7,0.0,49,147 +7,mRMR,Non-search,Information theory,1266.0,1212.0,1325.0,7.38,54.4,28.8,,0.507,0.753,0.3,0.0,49,147 +8,GainRatio,Non-search,Information theory,1258.0,1202.0,1315.0,7.79,51.5,29.8,,0.677,0.956,0.1,0.0,49,147 +9,SFS,Search,Forward search,1246.0,1186.0,1307.0,7.36,54.6,28.8,,0.359,0.751,761.8,31.3,49,147 +10,Lasso,Non-search,Regularization,1242.0,1185.0,1297.0,7.54,53.3,30.0,,0.383,0.863,2.4,0.0,49,147 +11,(R)ReliefF,Non-search,Distance,1208.0,1149.0,1267.0,8.43,46.9,32.9,,0.633,0.829,46.8,0.0,49,147 +12,LOCO,Search,Leave-one-out,1205.0,1148.0,1263.0,8.12,49.1,33.6,,0.298,0.648,215.1,0.0,49,147 +13,SBE,Search,Backward search,1188.0,1121.0,1250.0,8.37,47.4,35.4,,0.192,0.634,3188.3,49.0,49,147 +14,Random,Baseline,Baseline,1000.0,933.0,1057.0,11.39,25.8,48.7,,-0.001,0.511,0.0,0.0,49,147 +15,LaplacianScore,Non-search,Distance,962.0,855.0,1038.0,12.38,18.7,53.5,,0.592,0.851,0.2,0.0,49,147 diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_1/elo_explorer.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_1/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..954b5bb2858937c766fe37b29a76b4062e3947e2 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_1/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 2
Hover or tap a method for its values.
90010001100120013001400EloRandomLaplacianScoreLOCOSBESFS(R)ReliefFGainRatiomRMRLassoMarkovBlanketCARTElasticNetF-TestMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_1/leaderboard_table.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_1/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..4b35220e238ae3f560a190e76df605ff38ece325 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_1/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 2
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1331+61 / −556.7721.70.6310.6960.20.0
🥈2MIInformation theorynon-search1311+47 / −497.1924.40.6320.8150.50.0
🥉3F-TestCorrelationnon-search1304+52 / −497.0827.50.7100.9160.00.0
4ElasticNetRegularizationnon-search1295+57 / −566.9126.80.4760.84611.60.0
5CARTTreenon-search1294+56 / −526.9925.80.4350.6020.10.0
6MarkovBlanketBackward searchsearch1290+60 / −547.0427.70.3880.4170.20.0
7LassoRegularizationnon-search1272+64 / −607.2523.80.3600.8002.40.0
8mRMRInformation theorynon-search1269+54 / −547.8627.20.5600.6820.50.0
9GainRatioInformation theorynon-search1267+51 / −517.6730.60.6850.9180.10.0
10(R)ReliefFDistancenon-search1260+53 / −598.0328.00.6400.74846.10.0
11SFSForward searchsearch1251+59 / −617.6128.50.2680.6571,389.844.2
12SBEBackward searchsearch1226+72 / −707.9832.50.1780.5432,442.948.3
13LOCOLeave-one-outsearch1219+62 / −608.1031.00.2210.589217.70.0
14LaplacianScoreDistancenon-search1006+76 / −9912.1550.10.6550.8530.20.0
15RandomBaseline1000+63 / −8211.3745.6-0.0000.4990.00.0
diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_1/pareto.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_1/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..bdc7a66927fda75a7a1b4382bb4584a3ad32d421 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_1/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,33.3962,24.1514,49,True +MI,32.0465,52.4816,49,False +CART,31.5837,18.2272,49,True +F-Test,30.4288,1.0025,49,True +mRMR,29.8604,17.6604,49,False +MarkovBlanket,28.6263,25.2156,49,False +ElasticNet,27.6114,3082.0827,49,False +GainRatio,27.1389,4.7682,49,False +Lasso,26.8703,2717.2789,49,False +(R)ReliefF,26.8061,26789.0657,49,False +SFS,24.825,20026.3228,49,False +LOCO,18.3652,4826.5613,49,False +SBE,16.9887,27740.2803,49,False +Random,0.0,0.0029,49,True +LaplacianScore,-32.3454,4.6523,49,False diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_1/pareto.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..2bad213559ae0be134771e0a0aa57992b0c9dabd --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 2
Hover or tap a method for its values.
10-210-110010110210310405101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-TestmRMRMarkovBlanketElasticNetGainRatioLasso(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_1/website_leaderboard.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_1/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c86cf9cd2972a908299a10b68d7471bc7587ffda --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_1/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1331.0,1276.0,1392.0,6.77,58.8,21.7,,0.631,0.696,0.2,0.0,49,147 +2,MI,Non-search,Information theory,1311.0,1262.0,1358.0,7.19,55.8,24.4,,0.632,0.815,0.5,0.0,49,147 +3,F-Test,Non-search,Correlation,1304.0,1255.0,1356.0,7.08,56.5,27.5,,0.71,0.916,0.0,0.0,49,147 +4,ElasticNet,Non-search,Regularization,1295.0,1239.0,1352.0,6.91,57.8,26.8,,0.476,0.846,11.6,0.0,49,147 +5,CART,Non-search,Tree,1294.0,1242.0,1350.0,6.99,57.2,25.8,,0.435,0.602,0.1,0.0,49,147 +6,MarkovBlanket,Search,Backward search,1290.0,1236.0,1350.0,7.04,56.8,27.7,,0.388,0.417,0.2,0.0,49,147 +7,Lasso,Non-search,Regularization,1272.0,1212.0,1336.0,7.25,55.4,23.8,,0.36,0.8,2.4,0.0,49,147 +8,mRMR,Non-search,Information theory,1269.0,1215.0,1323.0,7.86,51.0,27.2,,0.56,0.682,0.5,0.0,49,147 +9,GainRatio,Non-search,Information theory,1267.0,1216.0,1318.0,7.67,52.3,30.6,,0.685,0.918,0.1,0.0,49,147 +10,(R)ReliefF,Non-search,Distance,1260.0,1201.0,1313.0,8.03,49.8,28.0,,0.64,0.748,46.1,0.0,49,147 +11,SFS,Search,Forward search,1251.0,1190.0,1310.0,7.61,52.8,28.5,,0.268,0.657,1389.8,44.2,49,147 +12,SBE,Search,Backward search,1226.0,1156.0,1298.0,7.98,50.2,32.5,,0.178,0.543,2442.9,48.3,49,147 +13,LOCO,Search,Leave-one-out,1219.0,1159.0,1281.0,8.1,49.3,31.0,,0.221,0.589,217.7,0.0,49,147 +14,LaplacianScore,Non-search,Distance,1006.0,907.0,1082.0,12.15,20.4,50.1,,0.655,0.853,0.2,0.0,49,147 +15,Random,Baseline,Baseline,1000.0,918.0,1063.0,11.37,26.0,45.6,,-0.0,0.499,0.0,0.0,49,147 diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_2/elo_explorer.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_2/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..ce2a5a9d44f72cc6bc38d50c9b03c3aff2bdff6c --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_2/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 3
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandomLOCOmRMRGainRatio(R)ReliefFLassoSFSSBEF-TestElasticNetMarkovBlanketCARTMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_2/leaderboard_table.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_2/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..c731ec2875b9ea1200751698b6021c3bba2000f4 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_2/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 3
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1298+60 / −496.5719.00.6680.6570.20.0
🥈2MIInformation theorynon-search1289+60 / −566.8422.50.6300.7740.50.0
🥉3CARTTreenon-search1283+58 / −596.6720.10.4350.5380.10.0
4MarkovBlanketBackward searchsearch1264+64 / −567.3224.60.4010.3980.20.0
5ElasticNetRegularizationnon-search1256+64 / −627.0223.20.4420.73811.60.0
6F-TestCorrelationnon-search1254+49 / −477.2024.20.6830.8640.00.0
7SBEBackward searchsearch1247+65 / −677.2026.30.1430.5572,411.546.9
8SFSForward searchsearch1212+60 / −587.7027.10.2140.6212,517.044.9
9LassoRegularizationnon-search1209+57 / −617.6224.30.3350.7022.40.0
10(R)ReliefFDistancenon-search1206+60 / −637.9323.70.6580.70646.40.0
11GainRatioInformation theorynon-search1198+56 / −488.3728.00.6730.8650.10.0
12mRMRInformation theorynon-search1184+54 / −528.4625.80.6130.6330.60.0
13LOCOLeave-one-outsearch1176+66 / −618.1127.90.1520.545226.68.2
14RandomBaseline1000+58 / −6910.7238.4-0.0000.5180.00.0
15LaplacianScoreDistancenon-search914+74 / −10512.2945.80.6560.8240.20.0
diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_2/pareto.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_2/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..79ae0c729103f41b5afe845e075dbaf77b273897 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_2/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,28.5587,24.0907,49,True +CART,26.0181,18.3597,49,True +MI,22.0869,52.728,49,False +F-Test,21.928,1.0192,49,True +MarkovBlanket,21.9166,25.0564,49,False +(R)ReliefF,21.0565,26740.7109,49,False +ElasticNet,19.771,2925.4613,49,False +mRMR,19.1363,36.9467,49,False +GainRatio,18.2231,4.8131,49,False +Lasso,17.6207,2688.9514,49,False +SBE,17.4836,25582.9557,49,False +SFS,15.9243,26022.5943,49,False +LOCO,15.3628,4924.8639,49,False +Random,0.0,0.0157,49,True +LaplacianScore,-287.7669,5.1408,49,False diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_2/pareto.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..3d1e927c6a2caf53b069db2c552b37924428d21d --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIF-TestMarkovBlanket(R)ReliefFElasticNetmRMRGainRatioLassoSBESFSLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_2/website_leaderboard.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_2/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..141dd0c12ff788bffa2a08d5b46ed680b269e102 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_2/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1298.0,1249.0,1358.0,6.57,60.2,19.0,,0.668,0.657,0.2,0.0,49,147 +2,MI,Non-search,Information theory,1289.0,1233.0,1349.0,6.84,58.3,22.5,,0.63,0.774,0.5,0.0,49,147 +3,CART,Non-search,Tree,1283.0,1224.0,1341.0,6.67,59.5,20.1,,0.435,0.538,0.1,0.0,49,147 +4,MarkovBlanket,Search,Backward search,1264.0,1208.0,1328.0,7.32,54.9,24.6,,0.401,0.398,0.2,0.0,49,147 +5,ElasticNet,Non-search,Regularization,1256.0,1194.0,1320.0,7.02,57.0,23.2,,0.442,0.738,11.6,0.0,49,147 +6,F-Test,Non-search,Correlation,1254.0,1207.0,1303.0,7.2,55.7,24.2,,0.683,0.864,0.0,0.0,49,147 +7,SBE,Search,Backward search,1247.0,1180.0,1312.0,7.2,55.7,26.3,,0.143,0.557,2411.5,46.9,49,147 +8,SFS,Search,Forward search,1212.0,1154.0,1272.0,7.7,52.2,27.1,,0.214,0.621,2517.0,44.9,49,147 +9,Lasso,Non-search,Regularization,1209.0,1148.0,1266.0,7.62,52.7,24.3,,0.335,0.702,2.4,0.0,49,147 +10,(R)ReliefF,Non-search,Distance,1206.0,1143.0,1266.0,7.93,50.5,23.7,,0.658,0.706,46.4,0.0,49,147 +11,GainRatio,Non-search,Information theory,1198.0,1150.0,1254.0,8.37,47.4,28.0,,0.673,0.865,0.1,0.0,49,147 +12,mRMR,Non-search,Information theory,1184.0,1132.0,1238.0,8.46,46.7,25.8,,0.613,0.633,0.6,0.0,49,147 +13,LOCO,Search,Leave-one-out,1176.0,1115.0,1242.0,8.11,49.2,27.9,,0.152,0.545,226.6,8.2,49,147 +14,Random,Baseline,Baseline,1000.0,931.0,1058.0,10.72,30.6,38.4,,-0.0,0.518,0.0,0.0,49,147 +15,LaplacianScore,Non-search,Distance,914.0,809.0,988.0,12.29,19.3,45.8,,0.656,0.824,0.2,0.0,49,147 diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_3/elo_explorer.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_3/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..01cfd0729e21700355f029edd14314c388faef9a --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_3/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 4
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandomLOCOGainRatiomRMR(R)ReliefFSFSLassoElasticNetMarkovBlanketMISBEF-TestCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_3/leaderboard_table.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_3/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..061d50f8208347980037fc10f547d8a5855a11b7 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_3/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 4
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1287+65 / −586.0613.70.6360.5760.20.0
🥈2CARTTreenon-search1242+51 / −466.8018.00.3920.4820.10.0
🥉3F-TestCorrelationnon-search1237+52 / −456.8816.90.6570.7730.00.0
4SBEBackward searchsearch1219+71 / −616.9421.00.0940.5231,642.745.6
5MIInformation theorynon-search1198+61 / −607.1321.80.6080.6950.50.0
6MarkovBlanketBackward searchsearch1170+63 / −558.1023.10.4190.4440.20.0
7ElasticNetRegularizationnon-search1168+68 / −637.7421.10.3730.62411.60.0
8LassoRegularizationnon-search1150+53 / −567.8621.50.2850.6152.40.0
9SFSForward searchsearch1148+51 / −547.7922.40.2170.5622,510.448.3
10(R)ReliefFDistancenon-search1138+58 / −658.0019.90.6410.62945.50.0
11GainRatioInformation theorynon-search1135+47 / −488.3422.90.6320.7700.10.0
12mRMRInformation theorynon-search1135+62 / −648.6419.80.6700.5780.78.2
13LOCOLeave-one-outsearch1114+62 / −558.1123.30.1080.520224.913.6
14RandomBaseline1000+58 / −649.8328.60.0020.4900.00.0
15LaplacianScoreDistancenon-search914+74 / −10411.8037.10.6830.7700.20.0
diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_3/pareto.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_3/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..b5c70e5e85219ebabd0e9a746d36c9fb42d3e5ac --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_3/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,20.6769,23.9021,49,True +CART,14.2797,18.1328,49,True +F-Test,11.42,1.0312,49,True +mRMR,9.7985,79.2957,49,False +MarkovBlanket,7.3607,25.0182,49,False +(R)ReliefF,7.1941,26781.6148,49,False +SBE,7.0163,20579.1548,49,False +GainRatio,5.6152,4.7477,49,False +LOCO,4.9346,5009.0202,49,False +SFS,4.7908,28425.4659,49,False +MI,3.8975,52.6355,49,False +Lasso,3.3826,2713.4054,49,False +ElasticNet,3.0344,2946.816,49,False +Random,0.0,0.0028,49,True +LaplacianScore,-56734689183369.57,4.7521,49,False diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_3/pareto.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..241941fbbc683326b8a20c7aa91c78455ea1c329 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 4
Hover or tap a method for its values.
10-210-110010110210310405101520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTF-TestmRMRMarkovBlanket(R)ReliefFSBEGainRatioLOCOSFSMILassoElasticNetRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_3/website_leaderboard.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_3/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..86de84ec69d2b2020c9200722436df63cdd519a5 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_3/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1287.0,1229.0,1352.0,6.06,63.8,13.7,,0.636,0.576,0.2,0.0,49,147 +2,CART,Non-search,Tree,1242.0,1196.0,1293.0,6.8,58.6,18.0,,0.392,0.482,0.1,0.0,49,147 +3,F-Test,Non-search,Correlation,1237.0,1192.0,1289.0,6.88,58.0,16.9,,0.657,0.773,0.0,0.0,49,147 +4,SBE,Search,Backward search,1219.0,1158.0,1290.0,6.94,57.6,21.0,,0.094,0.523,1642.7,45.6,49,147 +5,MI,Non-search,Information theory,1198.0,1138.0,1259.0,7.13,56.2,21.8,,0.608,0.695,0.5,0.0,49,147 +6,MarkovBlanket,Search,Backward search,1170.0,1115.0,1233.0,8.1,49.3,23.1,,0.419,0.444,0.2,0.0,49,147 +7,ElasticNet,Non-search,Regularization,1168.0,1105.0,1236.0,7.74,51.9,21.1,,0.373,0.624,11.6,0.0,49,147 +8,Lasso,Non-search,Regularization,1150.0,1094.0,1203.0,7.86,51.0,21.5,,0.285,0.615,2.4,0.0,49,147 +9,SFS,Search,Forward search,1148.0,1094.0,1199.0,7.79,51.5,22.4,,0.217,0.562,2510.4,48.3,49,147 +10,(R)ReliefF,Non-search,Distance,1138.0,1073.0,1196.0,8.0,50.0,19.9,,0.641,0.629,45.5,0.0,49,147 +11,GainRatio,Non-search,Information theory,1135.0,1087.0,1182.0,8.34,47.6,22.9,,0.632,0.77,0.1,0.0,49,147 +12,mRMR,Non-search,Information theory,1135.0,1071.0,1197.0,8.64,45.5,19.8,,0.67,0.578,0.7,8.2,49,147 +13,LOCO,Search,Leave-one-out,1114.0,1059.0,1176.0,8.11,49.2,23.3,,0.108,0.52,224.9,13.6,49,147 +14,Random,Baseline,Baseline,1000.0,936.0,1058.0,9.83,36.9,28.6,,0.002,0.49,0.0,0.0,49,147 +15,LaplacianScore,Non-search,Distance,914.0,810.0,988.0,11.8,22.9,37.1,,0.683,0.77,0.2,0.0,49,147 diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_4/elo_explorer.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_4/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..c077715dc351a44bbd5bf6e80accadf96af8128f --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_4/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 5
Hover or tap a method for its values.
90010001100120013001400EloLaplacianScoreRandomMarkovBlanketGainRatioLOCOLassomRMRSFSMISBECART(R)ReliefFElasticNetF-TestRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_4/leaderboard_table.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_4/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..f3072b7a19e3ed637a30359ee8b75b49b841f609 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_4/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 5
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1286+58 / −495.8711.20.5670.5000.20.0
🥈2F-TestCorrelationnon-search1202+69 / −537.3711.60.5490.6840.00.0
🥉3ElasticNetRegularizationnon-search1170+62 / −527.3513.10.2490.54111.60.0
4(R)ReliefFDistancenon-search1169+62 / −617.7814.60.5930.57245.50.0
5CARTTreenon-search1167+52 / −517.1614.50.2790.4750.10.0
6SBEBackward searchsearch1149+57 / −537.6116.20.0550.513775.942.5
7MIInformation theorynon-search1144+63 / −657.6516.10.5430.6110.50.0
8SFSForward searchsearch1135+57 / −617.4515.80.1630.5252,831.247.3
9mRMRInformation theorynon-search1120+59 / −578.0315.40.6860.5100.812.2
10LassoRegularizationnon-search1115+57 / −558.1014.80.1870.5442.40.0
11LOCOLeave-one-outsearch1102+60 / −558.0016.50.0760.513223.419.0
12GainRatioInformation theorynon-search1089+60 / −578.6618.00.5440.6740.10.0
13MarkovBlanketBackward searchsearch1088+58 / −628.6517.90.4000.4450.20.0
14RandomBaseline1000+60 / −729.4021.50.0020.5010.00.0
15LaplacianScoreDistancenon-search944+64 / −8610.8128.50.6830.7230.20.0
diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_4/pareto.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_4/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..60fad087143973ace5d2f3a57b4a11007df5696e --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_4/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,13.5236,24.3554,49,True +F-Test,12.8694,1.7137,49,True +CART,9.5783,18.4404,49,False +(R)ReliefF,9.2438,26808.442,49,False +ElasticNet,7.7759,3027.9142,49,False +LOCO,7.6053,5028.8987,49,False +GainRatio,6.5537,4.761,49,False +Lasso,6.5441,2673.3477,49,False +MarkovBlanket,4.7631,24.0607,49,False +SBE,4.1331,13589.7902,49,False +mRMR,3.2834,143.2032,49,False +SFS,2.3308,30108.5276,49,False +Random,0.0,0.0481,49,True +MI,-11346937836655.227,52.3447,49,False +LaplacianScore,-135431193534355.61,4.6989,49,False diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_4/pareto.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..af6ec123e0838d842ffe1639e75f1cd7d76adab6 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, TabICLv2, budget 5
Hover or tap a method for its values.
10-1100101102103104-1e+13-8e+12-6e+12-4e+12-2e+120Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestCART(R)ReliefFElasticNetLOCOGainRatioLassoMarkovBlanketSBEmRMRSFSRandomMIDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_4/website_leaderboard.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_4/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..f988b934c6bb2c9f30b714e4dd04971c87372b82 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_4/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1286.0,1237.0,1344.0,5.87,65.2,11.2,,0.567,0.5,0.2,0.0,49,147 +2,F-Test,Non-search,Correlation,1202.0,1149.0,1271.0,7.37,54.4,11.6,,0.549,0.684,0.0,0.0,49,147 +3,ElasticNet,Non-search,Regularization,1170.0,1118.0,1232.0,7.35,54.6,13.1,,0.249,0.541,11.6,0.0,49,147 +4,(R)ReliefF,Non-search,Distance,1169.0,1108.0,1231.0,7.78,51.5,14.6,,0.593,0.572,45.5,0.0,49,147 +5,CART,Non-search,Tree,1167.0,1116.0,1219.0,7.16,56.0,14.5,,0.279,0.475,0.1,0.0,49,147 +6,SBE,Search,Backward search,1149.0,1096.0,1206.0,7.61,52.8,16.2,,0.055,0.513,775.9,42.5,49,146 +7,MI,Non-search,Information theory,1144.0,1079.0,1207.0,7.65,52.4,16.1,,0.543,0.611,0.5,0.0,49,147 +8,SFS,Search,Forward search,1135.0,1074.0,1192.0,7.45,53.9,15.8,,0.163,0.525,2831.2,47.3,49,146 +9,mRMR,Non-search,Information theory,1120.0,1063.0,1179.0,8.03,49.7,15.4,,0.686,0.51,0.8,12.2,49,147 +10,Lasso,Non-search,Regularization,1115.0,1060.0,1172.0,8.1,49.2,14.8,,0.187,0.544,2.4,0.0,49,147 +11,LOCO,Search,Leave-one-out,1102.0,1047.0,1162.0,8.0,49.9,16.5,,0.076,0.513,223.4,19.0,49,147 +12,GainRatio,Non-search,Information theory,1089.0,1032.0,1149.0,8.66,45.2,18.0,,0.544,0.674,0.1,0.0,49,147 +13,MarkovBlanket,Search,Backward search,1088.0,1026.0,1146.0,8.65,45.3,17.9,,0.4,0.445,0.2,0.0,49,147 +14,Random,Baseline,Baseline,1000.0,928.0,1060.0,9.4,39.9,21.5,,0.002,0.501,0.0,0.0,49,147 +15,LaplacianScore,Non-search,Distance,944.0,858.0,1008.0,10.81,29.9,28.5,,0.683,0.723,0.2,0.0,49,147 diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_all/elo_explorer.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_all/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..cc615a245cfb5bc73e91d4addf7bf4afaccafebe --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_all/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: binary, TabICLv2, all budgets
Hover or tap a method for its values.
90010001100120013001400EloLaplacianScoreRandomLOCOGainRatiomRMR(R)ReliefFSFSLassoSBEMarkovBlanketElasticNetMICARTF-TestRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_all/leaderboard_table.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_all/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..7011e5024c751a489e27a462b638e74c9a4d4f99 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_all/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: binary, TabICLv2, all budgets
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1303+38 / −346.3217.90.6190.6370.20.0
🥈2F-TestCorrelationnon-search1254+36 / −347.0921.20.6590.8350.00.0
🥉3CARTTreenon-search1248+42 / −386.9421.30.3970.5570.10.0
4MIInformation theorynon-search1238+40 / −387.2022.20.6030.7500.50.0
5ElasticNetRegularizationnon-search1228+50 / −467.2122.50.4050.72911.60.0
6MarkovBlanketBackward searchsearch1218+48 / −427.5523.60.4020.4230.20.0
7SBEBackward searchsearch1202+51 / −507.6226.30.1320.5542,062.246.5
8SFSForward searchsearch1194+45 / −457.5824.50.2540.6231,839.043.2
9LassoRegularizationnon-search1194+41 / −417.6722.90.3100.7052.40.0
10(R)ReliefFDistancenon-search1192+43 / −468.0323.80.6330.69745.80.0
11mRMRInformation theorynon-search1191+35 / −378.0723.40.6040.6310.64.1
12GainRatioInformation theorynon-search1185+40 / −358.1725.80.6420.8360.10.0
13LOCOLeave-one-outsearch1160+49 / −488.0926.50.1710.563221.88.2
14RandomBaseline1000+43 / −4610.5436.60.0010.5040.00.0
15LaplacianScoreDistancenon-search948+61 / −7411.8943.00.6540.8040.20.0
diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_all/pareto.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_all/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..585d891b21898360dd1a1fd36a98d1238c869074 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_all/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,31.1346,24.1005,49,True +CART,28.1508,18.2594,49,True +F-Test,27.8012,1.1607,49,True +MI,27.3231,52.547,49,False +mRMR,25.6396,57.4379,49,False +MarkovBlanket,25.0141,24.9412,49,False +ElasticNet,23.3725,3014.2796,49,False +(R)ReliefF,23.3525,26762.4828,49,False +GainRatio,22.8399,4.7688,49,False +Lasso,22.3219,2694.7905,49,False +SFS,21.8469,23685.7071,49,False +LOCO,16.0356,4922.6654,49,False +SBE,14.3692,23471.438,49,False +Random,0.0,0.0145,49,True +LaplacianScore,-24.5419,4.8184,49,False diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_all/pareto.html b/data/subsets/tasks_binary/models_tabiclv2/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..7d9d2246a0b220d599b54ea0c25af77945791691 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: binary, TabICLv2, all budgets
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTF-TestMImRMRMarkovBlanketElasticNet(R)ReliefFGainRatioLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_binary/models_tabiclv2/budgets_all/website_leaderboard.csv b/data/subsets/tasks_binary/models_tabiclv2/budgets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..b7a25013e371e5af8ff405029e043bbada05b235 --- /dev/null +++ b/data/subsets/tasks_binary/models_tabiclv2/budgets_all/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1303.0,1269.0,1341.0,6.32,62.0,17.9,,0.619,0.637,0.2,0.0,49,735 +2,F-Test,Non-search,Correlation,1254.0,1220.0,1290.0,7.09,56.5,21.2,,0.659,0.835,0.0,0.0,49,735 +3,CART,Non-search,Tree,1248.0,1210.0,1290.0,6.94,57.6,21.3,,0.397,0.557,0.1,0.0,49,735 +4,MI,Non-search,Information theory,1238.0,1200.0,1278.0,7.2,55.7,22.2,,0.603,0.75,0.5,0.0,49,735 +5,ElasticNet,Non-search,Regularization,1228.0,1182.0,1278.0,7.21,55.6,22.5,,0.405,0.729,11.6,0.0,49,735 +6,MarkovBlanket,Search,Backward search,1218.0,1176.0,1266.0,7.55,53.2,23.6,,0.402,0.423,0.2,0.0,49,735 +7,SBE,Search,Backward search,1202.0,1152.0,1253.0,7.62,52.7,26.3,,0.132,0.554,2062.2,46.5,49,734 +8,SFS,Search,Forward search,1194.0,1149.0,1239.0,7.58,53.0,24.5,,0.254,0.623,1839.0,43.2,49,734 +9,Lasso,Non-search,Regularization,1194.0,1153.0,1235.0,7.67,52.3,22.9,,0.31,0.705,2.4,0.0,49,735 +10,(R)ReliefF,Non-search,Distance,1192.0,1146.0,1235.0,8.03,49.7,23.8,,0.633,0.697,45.8,0.0,49,735 +11,mRMR,Non-search,Information theory,1191.0,1154.0,1226.0,8.07,49.5,23.4,,0.604,0.631,0.6,4.1,49,735 +12,GainRatio,Non-search,Information theory,1185.0,1150.0,1225.0,8.17,48.8,25.8,,0.642,0.836,0.1,0.0,49,735 +13,LOCO,Search,Leave-one-out,1160.0,1112.0,1209.0,8.09,49.4,26.5,,0.171,0.563,221.8,8.2,49,735 +14,Random,Baseline,Baseline,1000.0,954.0,1043.0,10.54,31.8,36.6,,0.001,0.504,0.0,0.0,49,735 +15,LaplacianScore,Non-search,Distance,948.0,874.0,1009.0,11.89,22.2,43.0,,0.654,0.804,0.2,0.0,49,735 diff --git a/data/subsets/tasks_classification/models_all/budgets_0/pareto.html b/data/subsets/tasks_classification/models_all/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..d4ab39aadd3ec84929494e24ef6d9fdf3e3d321d --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, all models, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIF-TestCARTMarkovBlanketmRMRElasticNetLassoSFSGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_all/budgets_0/per_model.csv b/data/subsets/tasks_classification/models_all/budgets_0/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..4ebc84892a9fa9f8ccdf471c593cdd05fdc106f5 --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_0/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1344.0,1348.0,1,1352.0,2,1323.0,1,1338.0,1,1364.0,1 +2,CART,Non-search,Tree,1317.0,1331.0,2,1350.0,3,1277.0,3,1312.0,3,1321.0,3 +3,MarkovBlanket,Search,Backward search,1315.0,1320.0,3,1325.0,6,1269.0,5,1330.0,2,1337.0,2 +4,F-Test,Non-search,Correlation,1300.0,1277.0,7,1355.0,1,1270.0,4,1297.0,4,1306.0,4 +5,MI,Non-search,Information theory,1298.0,1288.0,5,1326.0,4,1290.0,2,1293.0,5,1301.0,5 +6,ElasticNet,Non-search,Regularization,1280.0,1285.0,6,1326.0,5,1232.0,8,1278.0,7,1288.0,6 +7,mRMR,Non-search,Information theory,1272.0,1265.0,8,1313.0,7,1225.0,9,1279.0,6,1284.0,7 +8,SFS,Search,Forward search,1268.0,1298.0,4,1291.0,9,1251.0,7,1264.0,8,1246.0,10 +9,GainRatio,Non-search,Information theory,1258.0,1244.0,10,1279.0,10,1251.0,6,1257.0,9,1266.0,8 +10,Lasso,Non-search,Regularization,1248.0,1255.0,9,1292.0,8,1199.0,11,1243.0,10,1256.0,9 +11,(R)ReliefF,Non-search,Distance,1215.0,1189.0,13,1235.0,11,1220.0,10,1209.0,11,1225.0,11 +12,SBE,Search,Backward search,1192.0,1223.0,11,1184.0,13,1161.0,12,1192.0,12,1203.0,13 +13,LOCO,Search,Leave-one-out,1185.0,1190.0,12,1210.0,12,1141.0,13,1177.0,13,1214.0,12 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,15,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,963.0,914.0,15,994.0,15,1004.0,14,929.0,15,960.0,15 diff --git a/data/subsets/tasks_classification/models_all/budgets_0/per_model_table.html b/data/subsets/tasks_classification/models_all/budgets_0/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..1e4b7eb1361aa7a3124d2fd51891e8f74c1d3098 --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_0/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, all models, budget 1 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1344🥇1348🥈1352🥇1323🥇1338🥇1364
🥈2CART1317🥈1331🥉1350🥉1277🥉1312🥉1321
🥉3MarkovBlanket1315🥉132013251269🥈1330🥈1337
4F-Test13001277🥇1355127012971306
5MI129812881326🥈129012931301
6ElasticNet128012851326123212781288
7mRMR127212651313122512791284
8SFS126812981291125112641246
9GainRatio125812441279125112571266
10Lasso124812551292119912431256
11(R)ReliefF121511891235122012091225
12SBE119212231184116111921203
13LOCO118511901210114111771214
14Random100010001000100010001000
15LaplacianScore9639149941004929960
diff --git a/data/subsets/tasks_classification/models_all/budgets_1/pareto.html b/data/subsets/tasks_classification/models_all/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..26455d49cfa0bba8c01f20d429806987a4aa428c --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, all models, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-TestElasticNetLassomRMRMarkovBlanketGainRatioSFS(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_all/budgets_1/per_model.csv b/data/subsets/tasks_classification/models_all/budgets_1/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..1bd7fbb1dca4a500163e1734ece2aa7054cd8959 --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_1/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1356.0,1360.0,2,1352.0,3,1352.0,1,1349.0,1,1376.0,1 +2,ElasticNet,Non-search,Regularization,1347.0,1364.0,1,1407.0,1,1315.0,2,1346.0,2,1318.0,6 +3,Lasso,Non-search,Regularization,1323.0,1340.0,5,1372.0,2,1299.0,3,1316.0,4,1299.0,7 +4,CART,Non-search,Tree,1323.0,1351.0,3,1303.0,7,1294.0,4,1340.0,3,1336.0,3 +5,MI,Non-search,Information theory,1319.0,1337.0,7,1340.0,4,1277.0,6,1303.0,6,1346.0,2 +6,MarkovBlanket,Search,Backward search,1305.0,1348.0,4,1288.0,8,1282.0,5,1294.0,7,1325.0,4 +7,F-Test,Non-search,Correlation,1303.0,1304.0,8,1326.0,5,1269.0,7,1307.0,5,1318.0,5 +8,mRMR,Non-search,Information theory,1287.0,1338.0,6,1315.0,6,1240.0,10,1257.0,11,1298.0,8 +9,GainRatio,Non-search,Information theory,1276.0,1304.0,9,1277.0,9,1262.0,8,1270.0,9,1277.0,10 +10,(R)ReliefF,Non-search,Distance,1262.0,1288.0,10,1218.0,11,1240.0,9,1289.0,8,1283.0,9 +11,SFS,Search,Forward search,1256.0,1286.0,11,1276.0,10,1221.0,12,1260.0,10,1246.0,11 +12,SBE,Search,Backward search,1218.0,1253.0,12,1181.0,13,1230.0,11,1210.0,13,1222.0,13 +13,LOCO,Search,Leave-one-out,1217.0,1243.0,13,1191.0,12,1189.0,13,1242.0,12,1224.0,12 +14,LaplacianScore,Non-search,Distance,1016.0,975.0,15,1017.0,14,1057.0,14,1006.0,14,1012.0,14 +15,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,15,1000.0,15,1000.0,15,1000.0,15 diff --git a/data/subsets/tasks_classification/models_all/budgets_1/per_model_table.html b/data/subsets/tasks_classification/models_all/budgets_1/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..d755897a63f6e67fb783db602b82cc853c4dfe7c --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_1/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, all models, budget 2 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1356🥈1360🥉1352🥇1352🥇1349🥇1376
🥈2ElasticNet1347🥇1364🥇1407🥈1315🥈13461318
🥉3Lasso13231340🥈1372🥉129913161299
4CART1323🥉135113031294🥉1340🥉1336
5MI13191337134012771303🥈1346
6MarkovBlanket130513481288128212941325
7F-Test130313041326126913071318
8mRMR128713381315124012571298
9GainRatio127613041277126212701277
10(R)ReliefF126212881218124012891283
11SFS125612861276122112601246
12SBE121812531181123012101222
13LOCO121712431191118912421224
14LaplacianScore10169751017105710061012
15Random100010001000100010001000
diff --git a/data/subsets/tasks_classification/models_all/budgets_2/pareto.html b/data/subsets/tasks_classification/models_all/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..2e32a800f731874e58bf1870d3e3b6ec1e00ab45 --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, all models, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTElasticNet(R)ReliefFF-TestLassoMarkovBlanketmRMRGainRatioSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_all/budgets_2/per_model.csv b/data/subsets/tasks_classification/models_all/budgets_2/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..2502761049b0fad430daca5ba718814c3608bc50 --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_2/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1300.0,1293.0,1,1235.0,4,1353.0,1,1317.0,1,1311.0,1 +2,ElasticNet,Non-search,Regularization,1274.0,1256.0,4,1310.0,1,1258.0,4,1301.0,2,1253.0,4 +3,MI,Non-search,Information theory,1271.0,1276.0,2,1256.0,3,1278.0,2,1264.0,4,1288.0,2 +4,CART,Non-search,Tree,1247.0,1247.0,5,1178.0,7,1262.0,3,1273.0,3,1281.0,3 +5,Lasso,Non-search,Regularization,1241.0,1264.0,3,1266.0,2,1230.0,6,1248.0,6,1206.0,9 +6,MarkovBlanket,Search,Backward search,1232.0,1239.0,6,1169.0,9,1250.0,5,1257.0,5,1253.0,5 +7,F-Test,Non-search,Correlation,1228.0,1221.0,7,1213.0,5,1226.0,7,1242.0,7,1242.0,6 +8,(R)ReliefF,Non-search,Distance,1194.0,1183.0,11,1162.0,11,1219.0,8,1196.0,9,1215.0,7 +9,GainRatio,Non-search,Information theory,1186.0,1197.0,9,1176.0,8,1172.0,10,1205.0,8,1183.0,11 +10,mRMR,Non-search,Information theory,1180.0,1194.0,10,1195.0,6,1164.0,11,1162.0,13,1191.0,10 +11,SBE,Search,Backward search,1175.0,1200.0,8,1098.0,13,1188.0,9,1185.0,10,1208.0,8 +12,SFS,Search,Forward search,1166.0,1178.0,12,1164.0,10,1154.0,12,1174.0,11,1163.0,12 +13,LOCO,Search,Leave-one-out,1137.0,1148.0,13,1118.0,12,1117.0,13,1162.0,12,1146.0,13 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,912.0,906.0,15,918.0,15,939.0,15,890.0,15,905.0,15 diff --git a/data/subsets/tasks_classification/models_all/budgets_2/per_model_table.html b/data/subsets/tasks_classification/models_all/budgets_2/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..58b14c0fd03d070822f1b5b6192e2537e05468b0 --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_2/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, all models, budget 3 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1300🥇12931235🥇1353🥇1317🥇1311
🥈2ElasticNet12741256🥇13101258🥈13011253
🥉3MI1271🥈1276🥉1256🥈12781264🥈1288
4CART124712471178🥉1262🥉1273🥉1281
5Lasso1241🥉1264🥈1266123012481206
6MarkovBlanket123212391169125012571253
7F-Test122812211213122612421242
8(R)ReliefF119411831162121911961215
9GainRatio118611971176117212051183
10mRMR118011941195116411621191
11SBE117512001098118811851208
12SFS116611781164115411741163
13LOCO113711481118111711621146
14Random100010001000100010001000
15LaplacianScore912906918939890905
diff --git a/data/subsets/tasks_classification/models_all/budgets_3/pareto.html b/data/subsets/tasks_classification/models_all/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..b1d0ec6950fee5931d86d7096cdba477394a7868 --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, all models, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMImRMRF-TestElasticNet(R)ReliefFLassoGainRatioMarkovBlanketSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_all/budgets_3/per_model.csv b/data/subsets/tasks_classification/models_all/budgets_3/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..bb0a6d6f2fd0c84e6dcfd49ba19e4829de137c9b --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_3/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1276.0,1253.0,1,1237.0,1,1307.0,1,1256.0,1,1334.0,1 +2,MI,Non-search,Information theory,1219.0,1184.0,5,1208.0,3,1223.0,4,1247.0,2,1237.0,3 +3,CART,Non-search,Tree,1214.0,1188.0,4,1140.0,7,1254.0,2,1243.0,3,1250.0,2 +4,ElasticNet,Non-search,Regularization,1210.0,1207.0,2,1221.0,2,1227.0,3,1214.0,4,1187.0,7 +5,F-Test,Non-search,Correlation,1188.0,1178.0,6,1183.0,5,1185.0,5,1178.0,6,1224.0,4 +6,Lasso,Non-search,Regularization,1180.0,1199.0,3,1205.0,4,1185.0,6,1153.0,9,1165.0,9 +7,MarkovBlanket,Search,Backward search,1157.0,1134.0,8,1095.0,13,1178.0,8,1190.0,5,1194.0,6 +8,SBE,Search,Backward search,1156.0,1112.0,10,1130.0,8,1170.0,10,1162.0,8,1209.0,5 +9,mRMR,Non-search,Information theory,1154.0,1156.0,7,1162.0,6,1176.0,9,1133.0,13,1145.0,11 +10,(R)ReliefF,Non-search,Distance,1150.0,1132.0,9,1098.0,12,1181.0,7,1169.0,7,1173.0,8 +11,GainRatio,Non-search,Information theory,1132.0,1110.0,11,1109.0,10,1151.0,11,1138.0,12,1153.0,10 +12,SFS,Search,Forward search,1114.0,1085.0,13,1129.0,9,1084.0,12,1142.0,11,1132.0,12 +13,LOCO,Search,Leave-one-out,1107.0,1093.0,12,1106.0,11,1081.0,13,1146.0,10,1109.0,13 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,922.0,865.0,15,963.0,15,933.0,15,909.0,15,930.0,15 diff --git a/data/subsets/tasks_classification/models_all/budgets_3/per_model_table.html b/data/subsets/tasks_classification/models_all/budgets_3/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..ef442ce3b01187afd437d5f3d78477d3e38feb6e --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_3/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, all models, budget 4 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1276🥇1253🥇1237🥇1307🥇1256🥇1334
🥈2MI12191184🥉12081223🥈1247🥉1237
🥉3CART121411881140🥈1254🥉1243🥈1250
4ElasticNet1210🥈1207🥈1221🥉122712141187
5F-Test118811781183118511781224
6Lasso1180🥉11991205118511531165
7MarkovBlanket115711341095117811901194
8SBE115611121130117011621209
9mRMR115411561162117611331145
10(R)ReliefF115011321098118111691173
11GainRatio113211101109115111381153
12SFS111410851129108411421132
13LOCO110710931106108111461109
14Random100010001000100010001000
15LaplacianScore922865963933909930
diff --git a/data/subsets/tasks_classification/models_all/budgets_4/pareto.html b/data/subsets/tasks_classification/models_all/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..63e02acd3a8e212a02c0990249aa08ad6e210da2 --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, all models, budget 5
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestmRMRElasticNet(R)ReliefFLassoCARTMIMarkovBlanketGainRatioLOCOSBESFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_all/budgets_4/per_model.csv b/data/subsets/tasks_classification/models_all/budgets_4/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..554a8a42710cd5969b32cb0545915001c1843a4b --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_4/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1253.0,1278.0,1,1217.0,1,1272.0,1,1202.0,1,1309.0,1 +2,CART,Non-search,Tree,1185.0,1168.0,2,1166.0,6,1217.0,2,1188.0,2,1188.0,3 +3,F-Test,Non-search,Correlation,1180.0,1166.0,3,1198.0,2,1198.0,3,1138.0,6,1205.0,2 +4,MI,Non-search,Information theory,1167.0,1148.0,6,1177.0,4,1186.0,4,1157.0,3,1169.0,6 +5,ElasticNet,Non-search,Regularization,1165.0,1166.0,4,1174.0,5,1179.0,5,1142.0,4,1170.0,5 +6,(R)ReliefF,Non-search,Distance,1150.0,1145.0,7,1121.0,9,1173.0,6,1139.0,5,1178.0,4 +7,mRMR,Non-search,Information theory,1133.0,1134.0,8,1179.0,3,1146.0,8,1090.0,9,1118.0,9 +8,Lasso,Non-search,Regularization,1128.0,1158.0,5,1152.0,7,1122.0,10,1089.0,10,1124.0,8 +9,MarkovBlanket,Search,Backward search,1115.0,1122.0,9,1121.0,8,1162.0,7,1062.0,12,1110.0,11 +10,SBE,Search,Backward search,1114.0,1105.0,11,1120.0,10,1115.0,11,1092.0,8,1138.0,7 +11,SFS,Search,Forward search,1095.0,1112.0,10,1113.0,12,1072.0,13,1069.0,11,1112.0,10 +12,GainRatio,Non-search,Information theory,1092.0,1089.0,12,1117.0,11,1134.0,9,1014.0,13,1106.0,12 +13,LOCO,Search,Leave-one-out,1090.0,1080.0,13,1085.0,13,1092.0,12,1099.0,7,1094.0,13 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,15,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,978.0,967.0,15,1041.0,14,953.0,15,963.0,15,961.0,15 diff --git a/data/subsets/tasks_classification/models_all/budgets_4/per_model_table.html b/data/subsets/tasks_classification/models_all/budgets_4/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..49e062c1df94b75afd5a29cebf62476cfec0ba1a --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_4/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, all models, budget 5 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1253🥇1278🥇1217🥇1272🥇1202🥇1309
🥈2CART1185🥈11681166🥈1217🥈1188🥉1188
🥉3F-Test1180🥉1166🥈1198🥉11981138🥈1205
4MI1167114811771186🥉11571169
5ElasticNet116511661174117911421170
6(R)ReliefF115011451121117311391178
7mRMR11331134🥉1179114610901118
8Lasso112811581152112210891124
9MarkovBlanket111511221121116210621110
10SBE111411051120111510921138
11SFS109511121113107210691112
12GainRatio109210891117113410141106
13LOCO109010801085109210991094
14Random100010001000100010001000
15LaplacianScore9789671041953963961
diff --git a/data/subsets/tasks_classification/models_all/budgets_all/pareto.html b/data/subsets/tasks_classification/models_all/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..a63cdd8b3d3a37c419b855a231f3feb7ba3774ab --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, all models, all budgets
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-TestmRMRElasticNetMarkovBlanketLassoGainRatio(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_all/budgets_all/per_model.csv b/data/subsets/tasks_classification/models_all/budgets_all/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..ea1182bc5ac6943593c0e592e25904313caa557d --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_all/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1300.0,1299.0,1,1271.0,2,1318.0,1,1285.0,1,1333.0,1 +2,CART,Non-search,Tree,1252.0,1249.0,2,1220.0,7,1258.0,2,1264.0,2,1270.0,2 +3,ElasticNet,Non-search,Regularization,1250.0,1248.0,3,1278.0,1,1239.0,4,1249.0,3,1238.0,5 +4,MI,Non-search,Information theory,1250.0,1239.0,4,1254.0,3,1248.0,3,1246.0,4,1263.0,3 +5,F-Test,Non-search,Correlation,1235.0,1222.0,7,1248.0,5,1227.0,5,1225.0,5,1254.0,4 +6,MarkovBlanket,Search,Backward search,1219.0,1225.0,6,1192.0,8,1225.0,6,1219.0,6,1238.0,6 +7,Lasso,Non-search,Regularization,1219.0,1236.0,5,1249.0,4,1204.0,7,1203.0,7,1205.0,8 +8,mRMR,Non-search,Information theory,1200.0,1210.0,8,1226.0,6,1187.0,10,1178.0,9,1203.0,9 +9,(R)ReliefF,Non-search,Distance,1190.0,1181.0,11,1160.0,11,1204.0,8,1194.0,8,1210.0,7 +10,GainRatio,Non-search,Information theory,1184.0,1182.0,10,1184.0,10,1191.0,9,1170.0,11,1192.0,10 +11,SFS,Search,Forward search,1175.0,1185.0,9,1187.0,9,1154.0,12,1175.0,10,1175.0,12 +12,SBE,Search,Backward search,1166.0,1172.0,12,1137.0,12,1170.0,11,1162.0,12,1192.0,11 +13,LOCO,Search,Leave-one-out,1142.0,1144.0,13,1136.0,13,1122.0,13,1159.0,13,1153.0,13 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,958.0,927.0,15,988.0,15,977.0,15,942.0,15,954.0,15 diff --git a/data/subsets/tasks_classification/models_all/budgets_all/per_model_table.html b/data/subsets/tasks_classification/models_all/budgets_all/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..6cdc0c8b44b48c7dee78b72c8d71d863d9fe9e82 --- /dev/null +++ b/data/subsets/tasks_classification/models_all/budgets_all/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, all models, all budgets · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1300🥇1299🥈1271🥇1318🥇1285🥇1333
🥈2CART1252🥈12491220🥈1258🥈1264🥈1270
🥉3ElasticNet1250🥉1248🥇12781239🥉12491238
4MI12501239🥉1254🥉12481246🥉1263
5F-Test123512221248122712251254
6MarkovBlanket121912251192122512191238
7Lasso121912361249120412031205
8mRMR120012101226118711781203
9(R)ReliefF119011811160120411941210
10GainRatio118411821184119111701192
11SFS117511851187115411751175
12SBE116611721137117011621192
13LOCO114211441136112211591153
14Random100010001000100010001000
15LaplacianScore958927988977942954
diff --git a/data/subsets/tasks_classification/models_lightgbm/budgets_0/pareto.html b/data/subsets/tasks_classification/models_lightgbm/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..d22c4028b123db1bb99684b066199a446cdb906c --- /dev/null +++ b/data/subsets/tasks_classification/models_lightgbm/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LightGBM, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTMarkovBlanketmRMRF-TestElasticNetSFSLassoGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_lightgbm/budgets_1/pareto.html b/data/subsets/tasks_classification/models_lightgbm/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..aa4d69c1811fdaf91e2a17adcae6e4c7a965c002 --- /dev/null +++ b/data/subsets/tasks_classification/models_lightgbm/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LightGBM, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTElasticNetLassomRMRMarkovBlanketF-TestGainRatio(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_lightgbm/budgets_2/pareto.html b/data/subsets/tasks_classification/models_lightgbm/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..6b347e543ce36208289b578522c5c6bb127e85c7 --- /dev/null +++ b/data/subsets/tasks_classification/models_lightgbm/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LightGBM, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTElasticNetLassoMarkovBlanketmRMRGainRatioF-Test(R)ReliefFSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_lightgbm/budgets_3/pareto.html b/data/subsets/tasks_classification/models_lightgbm/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..59757c01c2d86498c68a1c841d2c8a1c49c4f0f8 --- /dev/null +++ b/data/subsets/tasks_classification/models_lightgbm/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LightGBM, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestCARTLassomRMRMIElasticNet(R)ReliefFGainRatioMarkovBlanketSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_lightgbm/budgets_4/pareto.html b/data/subsets/tasks_classification/models_lightgbm/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..408af21c41e2be256cc6edac25db88eb34ddd091 --- /dev/null +++ b/data/subsets/tasks_classification/models_lightgbm/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LightGBM, budget 5
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestmRMRLassoCARTMarkovBlanket(R)ReliefFElasticNetSBEGainRatioMISFSLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_lightgbm/budgets_all/pareto.html b/data/subsets/tasks_classification/models_lightgbm/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..80c4a857045003d0cf3854629847a594652684e4 --- /dev/null +++ b/data/subsets/tasks_classification/models_lightgbm/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LightGBM, all budgets
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMImRMRElasticNetF-TestMarkovBlanketLassoGainRatio(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_linearmodel/budgets_0/pareto.html b/data/subsets/tasks_classification/models_linearmodel/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..1413e382a8ce254c0c7cccc19b7fcc1c9840f1f0 --- /dev/null +++ b/data/subsets/tasks_classification/models_linearmodel/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LinearModel, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalF-TestRFImportanceMICARTmRMRElasticNetMarkovBlanketLassoSFSGainRatio(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_linearmodel/budgets_1/pareto.html b/data/subsets/tasks_classification/models_linearmodel/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..feaaab1170bf38649689d048b15936c954fd498a --- /dev/null +++ b/data/subsets/tasks_classification/models_linearmodel/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LinearModel, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIElasticNetF-TestLassomRMRCARTGainRatioMarkovBlanketSFS(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_linearmodel/budgets_2/pareto.html b/data/subsets/tasks_classification/models_linearmodel/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..4ddd17219db876a64175da956188ae626a1a878b --- /dev/null +++ b/data/subsets/tasks_classification/models_linearmodel/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LinearModel, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIElasticNetLassoF-TestmRMR(R)ReliefFCARTGainRatioMarkovBlanketSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_linearmodel/budgets_3/pareto.html b/data/subsets/tasks_classification/models_linearmodel/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..1f9956d7b2e3bc3aee9007080eb93631bace5ecc --- /dev/null +++ b/data/subsets/tasks_classification/models_linearmodel/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LinearModel, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.5Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceElasticNetMILassomRMRF-TestCARTGainRatio(R)ReliefFSFSMarkovBlanketLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_linearmodel/budgets_4/pareto.html b/data/subsets/tasks_classification/models_linearmodel/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..50f9a3ee98e7b66fdfb0c16b892d3ad5dcabbdb5 --- /dev/null +++ b/data/subsets/tasks_classification/models_linearmodel/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LinearModel, budget 5
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportancemRMRElasticNetF-TestMarkovBlanketGainRatioMICARTLasso(R)ReliefFSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_linearmodel/budgets_all/pareto.html b/data/subsets/tasks_classification/models_linearmodel/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..57c7e9209c6b9a365b58b43c43c18c5ecd4db297 --- /dev/null +++ b/data/subsets/tasks_classification/models_linearmodel/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, LinearModel, all budgets
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIF-TestElasticNetmRMRCARTLassoMarkovBlanketGainRatioSFS(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_randomforest/budgets_0/pareto.html b/data/subsets/tasks_classification/models_randomforest/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..687695921d20eb093abc9dbae981ce417c41547a --- /dev/null +++ b/data/subsets/tasks_classification/models_randomforest/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RandomForest, budget 1
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalMIRFImportanceF-TestMarkovBlanketCARTGainRatioElasticNetmRMRSFSLasso(R)ReliefFLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_randomforest/budgets_1/pareto.html b/data/subsets/tasks_classification/models_randomforest/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..5f132b3d8d5b532ffd9bf19c5bec3d0a4490f1cb --- /dev/null +++ b/data/subsets/tasks_classification/models_randomforest/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RandomForest, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-TestMarkovBlanketElasticNetLassoGainRatio(R)ReliefFmRMRSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_randomforest/budgets_2/pareto.html b/data/subsets/tasks_classification/models_randomforest/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..8ff56c6a803230f4f587c211651ac1b62dbdf7a7 --- /dev/null +++ b/data/subsets/tasks_classification/models_randomforest/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RandomForest, budget 3
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketElasticNetCART(R)ReliefFF-TestLassomRMRGainRatioSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_randomforest/budgets_3/pareto.html b/data/subsets/tasks_classification/models_randomforest/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..f3e828e5c571b2cb7489256d418dc090cb392656 --- /dev/null +++ b/data/subsets/tasks_classification/models_randomforest/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RandomForest, budget 4
Hover or tap a method for its values.
10-210-110010110210310405101520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-Test(R)ReliefFElasticNetGainRatioLassoMarkovBlanketmRMRSBELOCOSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_randomforest/budgets_4/pareto.html b/data/subsets/tasks_classification/models_randomforest/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..6832f853bf555b677421075092f23cbed500e2f2 --- /dev/null +++ b/data/subsets/tasks_classification/models_randomforest/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RandomForest, budget 5
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestMI(R)ReliefFMarkovBlanketmRMRElasticNetCARTGainRatioSBELassoLOCOSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_randomforest/budgets_all/pareto.html b/data/subsets/tasks_classification/models_randomforest/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..f8bd30248e354209dbe3c21ddfa38be42f3fcde2 --- /dev/null +++ b/data/subsets/tasks_classification/models_randomforest/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RandomForest, all budgets
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTF-TestMarkovBlanketElasticNetmRMRLassoGainRatio(R)ReliefFSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_0/elo_explorer.html b/data/subsets/tasks_classification/models_realmlp/budgets_0/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..4e8114202ebc46229e780ebdd6684fa69a7d61a1 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_0/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, RealMLP, budget 1
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandomLOCOSBE(R)ReliefFLassoGainRatioSFSElasticNetmRMRMIF-TestCARTMarkovBlanketRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_0/leaderboard_table.html b/data/subsets/tasks_classification/models_realmlp/budgets_0/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..fa0efb84c5bca123d5b43bc7d55365dcad57cde5 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_0/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, RealMLP, budget 1
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1338+57 / −506.4624.00.6130.7990.20.0
🥈2MarkovBlanketBackward searchsearch1330+52 / −476.2625.50.4160.4270.20.0
🥉3CARTTreenon-search1312+59 / −476.5026.70.4620.7180.10.0
4F-TestCorrelationnon-search1297+49 / −457.0029.00.7110.9500.00.0
5MIInformation theorynon-search1293+47 / −497.1128.20.6290.8840.50.0
6mRMRInformation theorynon-search1279+47 / −457.3028.40.5290.7940.30.0
7ElasticNetRegularizationnon-search1278+55 / −557.0230.20.5070.91118.80.0
8SFSForward searchsearch1264+56 / −527.2427.90.3690.743846.629.5
9GainRatioInformation theorynon-search1257+48 / −457.8931.70.6860.9590.10.0
10LassoRegularizationnon-search1243+53 / −517.6230.90.4040.8854.20.0
11(R)ReliefFDistancenon-search1209+62 / −588.5232.50.6430.86212.80.0
12SBEBackward searchsearch1192+55 / −598.7034.80.1930.6233,600.351.9
13LOCOLeave-one-outsearch1177+46 / −508.4234.20.2970.662271.70.0
14RandomBaseline1000+56 / −6311.4547.4-0.0020.4960.00.0
15LaplacianScoreDistancenon-search929+82 / −11412.5252.00.5720.8500.20.0
diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_0/pareto.csv b/data/subsets/tasks_classification/models_realmlp/budgets_0/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..82c71444bc35165ba748b0dfcaa2cf937a7ab01d --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_0/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,33.1039,24.8218,61,True +F-Test,31.3952,1.1393,61,True +MI,30.9402,47.1743,61,False +CART,29.6821,16.6823,61,False +MarkovBlanket,29.2858,22.7637,61,False +mRMR,28.088,9.6333,61,False +SFS,26.3369,16611.6661,61,False +GainRatio,25.6301,4.6052,61,False +ElasticNet,25.6273,3319.5891,61,False +(R)ReliefF,24.6964,27520.8077,61,False +Lasso,24.2667,3414.5185,61,False +LOCO,16.5343,5862.9764,61,False +SBE,14.4957,34802.1959,61,False +Random,0.0,0.0369,61,True +LaplacianScore,-19.632,4.7408,61,False diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_0/pareto.html b/data/subsets/tasks_classification/models_realmlp/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..36e866f362357f4752dcd4b4b552378c35ffe08a --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RealMLP, budget 1
Hover or tap a method for its values.
10-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-TestMICARTMarkovBlanketmRMRSFSGainRatioElasticNet(R)ReliefFLassoLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_0/website_leaderboard.csv b/data/subsets/tasks_classification/models_realmlp/budgets_0/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..e5e90b4425fab72e5d6933e929ab247329f317a7 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_0/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1338.0,1288.0,1395.0,6.46,61.0,24.0,,0.613,0.799,0.2,0.0,61,183 +2,MarkovBlanket,Search,Backward search,1330.0,1283.0,1382.0,6.26,62.4,25.5,,0.416,0.427,0.2,0.0,61,183 +3,CART,Non-search,Tree,1312.0,1265.0,1371.0,6.5,60.7,26.7,,0.462,0.718,0.1,0.0,61,183 +4,F-Test,Non-search,Correlation,1297.0,1252.0,1346.0,7.0,57.2,29.0,,0.711,0.95,0.0,0.0,61,183 +5,MI,Non-search,Information theory,1293.0,1244.0,1340.0,7.11,56.3,28.2,,0.629,0.884,0.5,0.0,61,183 +6,mRMR,Non-search,Information theory,1279.0,1234.0,1326.0,7.3,55.0,28.4,,0.529,0.794,0.3,0.0,61,183 +7,ElasticNet,Non-search,Regularization,1278.0,1223.0,1333.0,7.02,57.0,30.2,,0.507,0.911,18.8,0.0,61,183 +8,SFS,Search,Forward search,1264.0,1212.0,1320.0,7.24,55.4,27.9,,0.369,0.743,846.6,29.5,61,183 +9,GainRatio,Non-search,Information theory,1257.0,1212.0,1305.0,7.89,50.8,31.7,,0.686,0.959,0.1,0.0,61,183 +10,Lasso,Non-search,Regularization,1243.0,1192.0,1296.0,7.62,52.7,30.9,,0.404,0.885,4.2,0.0,61,183 +11,(R)ReliefF,Non-search,Distance,1209.0,1151.0,1271.0,8.52,46.3,32.5,,0.643,0.862,12.8,0.0,61,183 +12,SBE,Search,Backward search,1192.0,1133.0,1247.0,8.7,45.0,34.8,,0.193,0.623,3600.3,51.9,61,183 +13,LOCO,Search,Leave-one-out,1177.0,1127.0,1223.0,8.42,47.0,34.2,,0.297,0.662,271.7,0.0,61,183 +14,Random,Baseline,Baseline,1000.0,937.0,1056.0,11.45,25.4,47.4,,-0.002,0.496,0.0,0.0,61,183 +15,LaplacianScore,Non-search,Distance,929.0,815.0,1011.0,12.52,17.7,52.0,,0.572,0.85,0.2,0.0,61,183 diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_1/elo_explorer.html b/data/subsets/tasks_classification/models_realmlp/budgets_1/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..46f9f62617aedc559529ff072089e2ad44319013 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_1/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, RealMLP, budget 2
Hover or tap a method for its values.
900100011001200130014001500EloRandomLaplacianScoreSBELOCOmRMRSFSGainRatio(R)ReliefFMarkovBlanketMIF-TestLassoCARTElasticNetRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_1/leaderboard_table.html b/data/subsets/tasks_classification/models_realmlp/budgets_1/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..44d9b8e94964e8cf24e854ccdb1259bbe24f403d --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_1/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, RealMLP, budget 2
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1349+49 / −466.5823.10.6430.7330.20.0
🥈2ElasticNetRegularizationnon-search1346+54 / −526.5826.10.5030.87018.80.0
🥉3CARTTreenon-search1340+51 / −446.6224.20.4410.6210.10.0
4LassoRegularizationnon-search1316+60 / −537.0027.60.3920.8244.30.0
5F-TestCorrelationnon-search1307+49 / −437.0928.00.7260.9310.00.0
6MIInformation theorynon-search1303+45 / −477.3428.60.6520.8480.40.0
7MarkovBlanketBackward searchsearch1294+50 / −457.1227.30.4040.4320.20.0
8(R)ReliefFDistancenon-search1289+61 / −577.6927.40.6520.79312.80.0
9GainRatioInformation theorynon-search1270+47 / −467.7629.90.6870.9290.10.0
10SFSForward searchsearch1260+50 / −507.9529.80.2810.6561,745.043.7
11mRMRInformation theorynon-search1257+47 / −457.9529.50.5840.7190.50.0
12LOCOLeave-one-outsearch1242+56 / −558.1630.80.2180.601274.70.0
13SBEBackward searchsearch1210+60 / −698.6433.70.1720.5533,600.150.3
14LaplacianScoreDistancenon-search1006+71 / −9312.0248.20.6340.8510.20.0
15RandomBaseline1000+48 / −6411.4944.70.0000.4960.00.0
diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_1/pareto.csv b/data/subsets/tasks_classification/models_realmlp/budgets_1/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..3e50a5e8ff1d0f36910599babb1a1907d9f33250 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_1/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,32.6251,24.8037,61,True +CART,30.1742,16.7194,61,True +MI,29.2847,46.6627,61,False +F-Test,28.947,0.9194,61,True +ElasticNet,27.369,3416.3524,61,False +GainRatio,26.7033,4.4944,61,False +(R)ReliefF,26.5529,27513.3731,61,False +MarkovBlanket,25.9019,22.6719,61,False +mRMR,25.7546,17.1813,61,False +Lasso,25.3804,3395.1795,61,False +SFS,23.4839,24246.0598,61,False +LOCO,18.1344,5954.9629,61,False +SBE,13.7032,31894.0239,61,False +Random,0.0,0.003,61,True +LaplacianScore,-33.2172,4.5257,61,False diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_1/pareto.html b/data/subsets/tasks_classification/models_realmlp/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..823207e00133b23d15637d7a75a98d684353138b --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RealMLP, budget 2
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIF-TestElasticNetGainRatio(R)ReliefFMarkovBlanketmRMRLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_1/website_leaderboard.csv b/data/subsets/tasks_classification/models_realmlp/budgets_1/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..bb958a8ea8cf3517c9a0023890f05fda3ae4e4b8 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_1/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1349.0,1303.0,1398.0,6.58,60.2,23.1,,0.643,0.733,0.2,0.0,61,183 +2,ElasticNet,Non-search,Regularization,1346.0,1294.0,1400.0,6.58,60.1,26.1,,0.503,0.87,18.8,0.0,61,183 +3,CART,Non-search,Tree,1340.0,1296.0,1391.0,6.62,59.9,24.2,,0.441,0.621,0.1,0.0,61,183 +4,Lasso,Non-search,Regularization,1316.0,1263.0,1376.0,7.0,57.1,27.6,,0.392,0.824,4.3,0.0,61,183 +5,F-Test,Non-search,Correlation,1307.0,1264.0,1356.0,7.09,56.5,28.0,,0.726,0.931,0.0,0.0,61,183 +6,MI,Non-search,Information theory,1303.0,1256.0,1348.0,7.34,54.7,28.6,,0.652,0.848,0.4,0.0,61,183 +7,MarkovBlanket,Search,Backward search,1294.0,1249.0,1344.0,7.12,56.3,27.3,,0.404,0.432,0.2,0.0,61,183 +8,(R)ReliefF,Non-search,Distance,1289.0,1232.0,1350.0,7.69,52.2,27.4,,0.652,0.793,12.8,0.0,61,183 +9,GainRatio,Non-search,Information theory,1270.0,1224.0,1317.0,7.76,51.7,29.9,,0.687,0.929,0.1,0.0,61,183 +10,SFS,Search,Forward search,1260.0,1210.0,1310.0,7.95,50.3,29.8,,0.281,0.656,1745.0,43.7,61,183 +11,mRMR,Non-search,Information theory,1257.0,1212.0,1304.0,7.95,50.4,29.5,,0.584,0.719,0.5,0.0,61,183 +12,LOCO,Search,Leave-one-out,1242.0,1187.0,1298.0,8.16,48.8,30.8,,0.218,0.601,274.7,0.0,61,183 +13,SBE,Search,Backward search,1210.0,1141.0,1270.0,8.64,45.4,33.7,,0.172,0.553,3600.1,50.3,61,183 +14,LaplacianScore,Non-search,Distance,1006.0,913.0,1077.0,12.02,21.2,48.2,,0.634,0.851,0.2,0.0,61,183 +15,Random,Baseline,Baseline,1000.0,936.0,1048.0,11.49,25.1,44.7,,0.0,0.496,0.0,0.0,61,183 diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_2/elo_explorer.html b/data/subsets/tasks_classification/models_realmlp/budgets_2/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..dbbdf7bbdd12f09dc3dff0cb9f87e328768b5d91 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_2/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, RealMLP, budget 3
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandomLOCOmRMRSFSSBE(R)ReliefFGainRatioF-TestLassoMarkovBlanketMICARTElasticNetRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_2/leaderboard_table.html b/data/subsets/tasks_classification/models_realmlp/budgets_2/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..3be72dfefd6ba567a5f3c75fdff9d5eaee91dfc0 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_2/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, RealMLP, budget 3
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1317+56 / −536.3018.80.6760.6890.20.0
🥈2ElasticNetRegularizationnon-search1301+58 / −516.2923.50.4700.76919.00.0
🥉3CARTTreenon-search1273+47 / −466.8023.00.4340.5590.10.0
4MIInformation theorynon-search1264+50 / −536.8223.90.6490.8020.40.0
5MarkovBlanketBackward searchsearch1257+52 / −447.1925.70.4170.4160.20.0
6LassoRegularizationnon-search1248+65 / −557.1124.10.3740.7234.30.0
7F-TestCorrelationnon-search1242+46 / −457.4025.60.6970.8790.00.0
8GainRatioInformation theorynon-search1205+46 / −488.0928.20.6850.8710.10.0
9(R)ReliefFDistancenon-search1196+60 / −588.0425.80.6680.74512.70.0
10SBEBackward searchsearch1185+53 / −538.1431.00.1420.5472,782.549.2
11SFSForward searchsearch1174+51 / −507.9927.00.2360.6243,129.146.4
12LOCOLeave-one-outsearch1162+57 / −528.4528.90.1500.560302.28.7
13mRMRInformation theorynon-search1162+43 / −438.2627.00.6240.6690.60.0
14RandomBaseline1000+56 / −6110.8238.3-0.0000.5170.00.0
15LaplacianScoreDistancenon-search890+74 / −10512.3045.60.6450.8240.20.0
diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_2/pareto.csv b/data/subsets/tasks_classification/models_realmlp/budgets_2/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..b223c3d8415b2ca1ebf0f8b747ce4508bf17b344 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_2/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,26.3886,24.8992,61,True +CART,22.5018,16.6886,61,True +MarkovBlanket,21.2499,22.8143,61,False +MI,20.2392,47.0889,61,False +GainRatio,19.271,4.5115,61,True +ElasticNet,18.1111,3368.4055,61,False +(R)ReliefF,18.0672,27521.1688,61,False +F-Test,17.9525,0.9691,61,True +Lasso,15.1524,3458.1475,61,False +mRMR,15.1495,35.0172,61,False +SFS,11.5995,30316.4184,61,False +SBE,9.539,28780.3979,61,False +LOCO,9.1037,6102.0591,61,False +Random,0.0,0.0034,61,True +LaplacianScore,-125.934,5.0751,61,False diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_2/pareto.html b/data/subsets/tasks_classification/models_realmlp/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..d77682ebb45529929c57feac86a3c7a62c5d91c7 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RealMLP, budget 3
Hover or tap a method for its values.
10-210-11001011021031040510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMarkovBlanketMIGainRatioElasticNet(R)ReliefFF-TestLassomRMRSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_2/website_leaderboard.csv b/data/subsets/tasks_classification/models_realmlp/budgets_2/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..7739754b02ca5f4fcc3cf433a456cd1898c89cfa --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_2/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1317.0,1264.0,1373.0,6.3,62.1,18.8,,0.676,0.689,0.2,0.0,61,183 +2,ElasticNet,Non-search,Regularization,1301.0,1250.0,1359.0,6.29,62.2,23.5,,0.47,0.769,19.0,0.0,61,183 +3,CART,Non-search,Tree,1273.0,1227.0,1320.0,6.8,58.6,23.0,,0.434,0.559,0.1,0.0,61,183 +4,MI,Non-search,Information theory,1264.0,1211.0,1314.0,6.82,58.4,23.9,,0.649,0.802,0.4,0.0,61,183 +5,MarkovBlanket,Search,Backward search,1257.0,1213.0,1309.0,7.19,55.8,25.7,,0.417,0.416,0.2,0.0,61,183 +6,Lasso,Non-search,Regularization,1248.0,1193.0,1313.0,7.11,56.3,24.1,,0.374,0.723,4.3,0.0,61,183 +7,F-Test,Non-search,Correlation,1242.0,1197.0,1288.0,7.4,54.3,25.6,,0.697,0.879,0.0,0.0,61,183 +8,GainRatio,Non-search,Information theory,1205.0,1157.0,1251.0,8.09,49.3,28.2,,0.685,0.871,0.1,0.0,61,183 +9,(R)ReliefF,Non-search,Distance,1196.0,1138.0,1256.0,8.04,49.7,25.8,,0.668,0.745,12.7,0.0,61,183 +10,SBE,Search,Backward search,1185.0,1132.0,1238.0,8.14,49.0,31.0,,0.142,0.547,2782.5,49.2,61,183 +11,SFS,Search,Forward search,1174.0,1124.0,1225.0,7.99,50.1,27.0,,0.236,0.624,3129.1,46.4,61,183 +12,LOCO,Search,Leave-one-out,1162.0,1110.0,1219.0,8.45,46.8,28.9,,0.15,0.56,302.2,8.7,61,183 +13,mRMR,Non-search,Information theory,1162.0,1119.0,1205.0,8.26,48.1,27.0,,0.624,0.669,0.6,0.0,61,183 +14,Random,Baseline,Baseline,1000.0,939.0,1056.0,10.82,29.8,38.3,,-0.0,0.517,0.0,0.0,61,183 +15,LaplacianScore,Non-search,Distance,890.0,785.0,964.0,12.3,19.3,45.6,,0.645,0.824,0.2,0.0,61,183 diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_3/elo_explorer.html b/data/subsets/tasks_classification/models_realmlp/budgets_3/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..276ff35f6c9aa73465df44ef6b1fea108964c085 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_3/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, RealMLP, budget 4
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandommRMRGainRatioSFSLOCOLassoSBE(R)ReliefFF-TestMarkovBlanketElasticNetCARTMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_3/leaderboard_table.html b/data/subsets/tasks_classification/models_realmlp/budgets_3/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..ef3b2883312abebe5afdc94335ba40a73d4bed55 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_3/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, RealMLP, budget 4
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1256+52 / −486.5217.30.6510.5990.20.0
🥈2MIInformation theorynon-search1247+53 / −486.6318.80.6280.7160.50.0
🥉3CARTTreenon-search1243+52 / −526.6318.70.3890.5020.10.0
4ElasticNetRegularizationnon-search1214+62 / −547.0420.70.4100.65718.90.0
5MarkovBlanketBackward searchsearch1190+49 / −487.7821.80.4360.4510.20.0
6F-TestCorrelationnon-search1178+48 / −517.3420.40.6610.7810.00.0
7(R)ReliefFDistancenon-search1169+50 / −537.9922.60.6460.65712.60.0
8SBEBackward searchsearch1162+58 / −637.8024.30.0960.5212,497.145.4
9LassoRegularizationnon-search1153+52 / −527.6822.50.3220.6344.30.0
10LOCOLeave-one-outsearch1146+63 / −538.1623.60.1170.531269.213.7
11SFSForward searchsearch1142+46 / −488.3223.60.2260.5633,600.050.8
12GainRatioInformation theorynon-search1138+49 / −508.4522.60.6520.7660.10.0
13mRMRInformation theorynon-search1133+49 / −458.2522.10.6710.6000.78.2
14RandomBaseline1000+54 / −599.6729.00.0020.4900.00.0
15LaplacianScoreDistancenon-search909+74 / −8511.7638.30.6790.7630.20.0
diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_3/pareto.csv b/data/subsets/tasks_classification/models_realmlp/budgets_3/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..643f993d5b3fad4d9c7ad50a7c79427d02e59a09 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_3/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,17.1045,25.5079,60,True +CART,15.9047,17.0771,60,True +MI,13.6276,47.9643,60,False +mRMR,11.6426,72.3061,60,False +GainRatio,11.5697,4.53,60,True +F-Test,11.3788,0.9309,60,True +MarkovBlanket,11.003,21.6757,60,False +(R)ReliefF,10.9574,27961.5818,60,False +LOCO,10.3622,6077.767,60,False +ElasticNet,9.6742,3316.6425,60,False +SBE,8.3222,24345.239,60,False +SFS,7.6997,33469.5191,60,False +Lasso,7.0207,3425.0332,60,False +Random,0.0,0.0032,60,True +LaplacianScore,-48.8485,4.559,60,False diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_3/pareto.html b/data/subsets/tasks_classification/models_realmlp/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..f5bb83be43f4c9f060ebe79a86ed263999c450ba --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RealMLP, budget 4
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.5Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMImRMRGainRatioF-TestMarkovBlanket(R)ReliefFLOCOElasticNetSBESFSLassoRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_3/website_leaderboard.csv b/data/subsets/tasks_classification/models_realmlp/budgets_3/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..c922961d11f26eb0bdaa7c986c1570ef8ad16d89 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_3/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1256.0,1208.0,1308.0,6.52,60.6,17.3,,0.651,0.599,0.2,0.0,61,183 +2,MI,Non-search,Information theory,1247.0,1199.0,1300.0,6.63,59.8,18.8,,0.628,0.716,0.5,0.0,61,183 +3,CART,Non-search,Tree,1243.0,1191.0,1295.0,6.63,59.8,18.7,,0.389,0.502,0.1,0.0,61,183 +4,ElasticNet,Non-search,Regularization,1214.0,1160.0,1276.0,7.04,56.9,20.7,,0.41,0.657,18.9,0.0,61,183 +5,MarkovBlanket,Search,Backward search,1190.0,1142.0,1239.0,7.78,51.6,21.8,,0.436,0.451,0.2,0.0,61,183 +6,F-Test,Non-search,Correlation,1178.0,1127.0,1226.0,7.34,54.7,20.4,,0.661,0.781,0.0,0.0,61,183 +7,(R)ReliefF,Non-search,Distance,1169.0,1116.0,1219.0,7.99,50.1,22.6,,0.646,0.657,12.6,0.0,61,183 +8,SBE,Search,Backward search,1162.0,1099.0,1220.0,7.8,51.4,24.3,,0.096,0.521,2497.1,45.4,61,183 +9,Lasso,Non-search,Regularization,1153.0,1101.0,1205.0,7.68,52.3,22.5,,0.322,0.634,4.3,0.0,61,183 +10,LOCO,Search,Leave-one-out,1146.0,1093.0,1209.0,8.16,48.9,23.6,,0.117,0.531,269.2,13.7,61,183 +11,SFS,Search,Forward search,1142.0,1094.0,1188.0,8.32,47.7,23.6,,0.226,0.563,3600.0,50.8,61,183 +12,GainRatio,Non-search,Information theory,1138.0,1088.0,1187.0,8.45,46.8,22.6,,0.652,0.766,0.1,0.0,61,183 +13,mRMR,Non-search,Information theory,1133.0,1088.0,1182.0,8.25,48.2,22.1,,0.671,0.6,0.7,8.2,61,183 +14,Random,Baseline,Baseline,1000.0,941.0,1054.0,9.67,38.1,29.0,,0.002,0.49,0.0,0.0,61,183 +15,LaplacianScore,Non-search,Distance,909.0,824.0,983.0,11.76,23.2,38.3,,0.679,0.763,0.2,0.0,61,183 diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_4/elo_explorer.html b/data/subsets/tasks_classification/models_realmlp/budgets_4/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..b1ac7aac48d1bb638f165526d6fd6c3dc9a94f79 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_4/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, RealMLP, budget 5
Hover or tap a method for its values.
9001000110012001300EloLaplacianScoreRandomGainRatioMarkovBlanketSFSLassomRMRSBELOCOF-Test(R)ReliefFElasticNetMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_4/leaderboard_table.html b/data/subsets/tasks_classification/models_realmlp/budgets_4/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..09e8feeafc7b098225f1ce4a79942cb4c554425c --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_4/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, RealMLP, budget 5
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1202+52 / −506.4916.80.5820.5110.20.0
🥈2CARTTreenon-search1188+51 / −506.4716.80.2980.4840.10.0
🥉3MIInformation theorynon-search1157+48 / −477.2418.40.5620.6240.40.0
4ElasticNetRegularizationnon-search1142+54 / −547.3318.60.2870.56118.80.0
5(R)ReliefFDistancenon-search1139+59 / −557.8018.90.6050.58812.60.0
6F-TestCorrelationnon-search1138+49 / −517.2817.90.5640.6890.00.0
7LOCOLeave-one-outsearch1099+43 / −507.9820.50.0840.514260.619.7
8SBEBackward searchsearch1092+48 / −528.0520.70.0590.5131,290.143.2
9mRMRInformation theorynon-search1090+48 / −497.9918.80.6630.5320.811.5
10LassoRegularizationnon-search1089+53 / −497.6819.40.2220.5574.30.0
11SFSForward searchsearch1069+50 / −558.4721.80.1860.5233,600.050.8
12MarkovBlanketBackward searchsearch1062+47 / −508.3121.00.4000.4500.20.0
13GainRatioInformation theorynon-search1014+45 / −509.1521.80.5560.6730.10.0
14RandomBaseline1000+53 / −589.5324.80.0020.5010.00.0
15LaplacianScoreDistancenon-search963+70 / −6910.2328.00.6670.7110.20.0
diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_4/pareto.csv b/data/subsets/tasks_classification/models_realmlp/budgets_4/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..9f7783ee9807c983d96000e44b97afbf0a1725ac --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_4/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,8.5455,25.0566,61,True +CART,8.4417,16.944,61,True +F-Test,7.5816,0.9194,61,True +(R)ReliefF,6.7428,27495.6889,61,False +ElasticNet,6.5116,3277.9953,61,False +mRMR,5.9319,134.2415,61,False +LOCO,5.3669,5859.8152,61,False +SBE,5.1958,15753.1898,61,False +MarkovBlanket,4.8377,21.6159,61,False +MI,3.4422,46.6719,61,False +Lasso,3.075,3411.6046,61,False +SFS,1.327,34559.05,61,False +GainRatio,0.4438,4.4903,61,False +Random,0.0,0.003,61,True +LaplacianScore,-63803273409434.43,4.846,61,False diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_4/pareto.html b/data/subsets/tasks_classification/models_realmlp/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..47e8b615d6a160c8e6d88e10d57455bc623ce2f4 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RealMLP, budget 5
Hover or tap a method for its values.
10-210-110010110210310402468Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTF-Test(R)ReliefFElasticNetmRMRLOCOSBEMarkovBlanketMILassoSFSGainRatioRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_4/website_leaderboard.csv b/data/subsets/tasks_classification/models_realmlp/budgets_4/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..7809724e4f47788f4248c56252b19e1d58c308f2 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_4/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1202.0,1152.0,1254.0,6.49,60.8,16.8,,0.582,0.511,0.2,0.0,61,183 +2,CART,Non-search,Tree,1188.0,1138.0,1239.0,6.47,60.9,16.8,,0.298,0.484,0.1,0.0,61,183 +3,MI,Non-search,Information theory,1157.0,1110.0,1205.0,7.24,55.4,18.4,,0.562,0.624,0.4,0.0,61,183 +4,ElasticNet,Non-search,Regularization,1142.0,1088.0,1196.0,7.33,54.8,18.6,,0.287,0.561,18.8,0.0,61,183 +5,(R)ReliefF,Non-search,Distance,1139.0,1084.0,1198.0,7.8,51.4,18.9,,0.605,0.588,12.6,0.0,61,183 +6,F-Test,Non-search,Correlation,1138.0,1087.0,1187.0,7.28,55.1,17.9,,0.564,0.689,0.0,0.0,61,183 +7,LOCO,Search,Leave-one-out,1099.0,1049.0,1142.0,7.98,50.2,20.5,,0.084,0.514,260.6,19.7,61,183 +8,SBE,Search,Backward search,1092.0,1040.0,1140.0,8.05,49.6,20.7,,0.059,0.513,1290.1,43.2,61,183 +9,mRMR,Non-search,Information theory,1090.0,1041.0,1138.0,7.99,50.0,18.8,,0.663,0.532,0.8,11.5,61,183 +10,Lasso,Non-search,Regularization,1089.0,1040.0,1142.0,7.68,52.3,19.4,,0.222,0.557,4.3,0.0,61,183 +11,SFS,Search,Forward search,1069.0,1014.0,1119.0,8.47,46.7,21.8,,0.186,0.523,3600.0,50.8,61,183 +12,MarkovBlanket,Search,Backward search,1062.0,1012.0,1109.0,8.31,47.8,21.0,,0.4,0.45,0.2,0.0,61,183 +13,GainRatio,Non-search,Information theory,1014.0,964.0,1059.0,9.15,41.8,21.8,,0.556,0.673,0.1,0.0,61,183 +14,Random,Baseline,Baseline,1000.0,942.0,1053.0,9.53,39.1,24.8,,0.002,0.501,0.0,0.0,61,183 +15,LaplacianScore,Non-search,Distance,963.0,894.0,1033.0,10.23,34.1,28.0,,0.667,0.711,0.2,0.0,61,183 diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_all/elo_explorer.html b/data/subsets/tasks_classification/models_realmlp/budgets_all/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..b45b24ebd9bbf7d1ca8ca2d0ed4506ec2d41e3aa --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_all/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, RealMLP, all budgets
Hover or tap a method for its values.
90010001100120013001400EloLaplacianScoreRandomLOCOSBEGainRatioSFSmRMR(R)ReliefFLassoMarkovBlanketF-TestMIElasticNetCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_all/leaderboard_table.html b/data/subsets/tasks_classification/models_realmlp/budgets_all/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..1ea65dc33d18df4821c67c716140575104753ffe --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_all/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, RealMLP, all budgets
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1285+35 / −346.4720.00.6330.6660.20.0
🥈2CARTTreenon-search1264+37 / −356.6021.90.4050.5770.10.0
🥉3ElasticNetRegularizationnon-search1249+41 / −396.8523.80.4350.75418.80.0
4MIInformation theorynon-search1246+31 / −347.0323.60.6240.7750.40.0
5F-TestCorrelationnon-search1225+34 / −337.2224.20.6720.8460.00.0
6MarkovBlanketBackward searchsearch1219+35 / −337.3324.30.4150.4350.20.0
7LassoRegularizationnon-search1203+39 / −387.4224.90.3430.7254.30.0
8(R)ReliefFDistancenon-search1194+41 / −408.0125.40.6430.72912.70.0
9mRMRInformation theorynon-search1178+33 / −317.9525.10.6120.6630.63.9
10SFSForward searchsearch1175+39 / −357.9926.00.2690.6222,260.144.3
11GainRatioInformation theorynon-search1170+33 / −328.2726.80.6530.8400.10.0
12SBEBackward searchsearch1162+41 / −438.2628.90.1310.5512,744.248.0
13LOCOLeave-one-outsearch1159+41 / −388.2327.60.1730.574272.58.4
14RandomBaseline1000+37 / −4010.5936.80.0010.5000.00.0
15LaplacianScoreDistancenon-search942+55 / −6711.7642.40.6390.8000.20.0
diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_all/pareto.csv b/data/subsets/tasks_classification/models_realmlp/budgets_all/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..649060d7078215b197325055acae805a652b53c0 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_all/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,29.4994,24.9347,61,True +CART,27.1325,16.7668,61,True +MI,26.4956,46.9585,61,False +F-Test,25.606,0.973,61,True +mRMR,24.0072,54.2181,61,False +MarkovBlanket,23.9236,22.4829,61,False +ElasticNet,23.0426,3329.0055,61,False +(R)ReliefF,22.7314,27510.8475,61,False +GainRatio,22.4104,4.5191,61,False +Lasso,21.0635,3409.6683,61,False +SFS,20.6096,27731.5859,61,False +LOCO,16.0653,5952.3742,61,False +SBE,13.3407,27035.9767,61,False +Random,0.0,0.0099,61,True +LaplacianScore,-21.1718,4.7349,61,False diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_all/pareto.html b/data/subsets/tasks_classification/models_realmlp/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..da62fb8370452151a5f4789cf7cc1988017db1f8 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, RealMLP, all budgets
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIF-TestmRMRMarkovBlanketElasticNet(R)ReliefFGainRatioLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_realmlp/budgets_all/website_leaderboard.csv b/data/subsets/tasks_classification/models_realmlp/budgets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..4c50d7d6b091ddc6b2dccb6e906940a445b58d84 --- /dev/null +++ b/data/subsets/tasks_classification/models_realmlp/budgets_all/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1285.0,1251.0,1320.0,6.47,60.9,20.0,,0.633,0.666,0.2,0.0,61,915 +2,CART,Non-search,Tree,1264.0,1229.0,1301.0,6.6,60.0,21.9,,0.405,0.577,0.1,0.0,61,915 +3,ElasticNet,Non-search,Regularization,1249.0,1210.0,1290.0,6.85,58.2,23.8,,0.435,0.754,18.8,0.0,61,915 +4,MI,Non-search,Information theory,1246.0,1212.0,1277.0,7.03,56.9,23.6,,0.624,0.775,0.4,0.0,61,915 +5,F-Test,Non-search,Correlation,1225.0,1192.0,1259.0,7.22,55.6,24.2,,0.672,0.846,0.0,0.0,61,915 +6,MarkovBlanket,Search,Backward search,1219.0,1186.0,1254.0,7.33,54.8,24.3,,0.415,0.435,0.2,0.0,61,915 +7,Lasso,Non-search,Regularization,1203.0,1165.0,1242.0,7.42,54.2,24.9,,0.343,0.725,4.3,0.0,61,915 +8,(R)ReliefF,Non-search,Distance,1194.0,1154.0,1235.0,8.01,50.0,25.4,,0.643,0.729,12.7,0.0,61,915 +9,mRMR,Non-search,Information theory,1178.0,1147.0,1211.0,7.95,50.4,25.1,,0.612,0.663,0.6,3.9,61,915 +10,SFS,Search,Forward search,1175.0,1140.0,1214.0,7.99,50.0,26.0,,0.269,0.622,2260.1,44.3,61,915 +11,GainRatio,Non-search,Information theory,1170.0,1138.0,1203.0,8.27,48.1,26.8,,0.653,0.84,0.1,0.0,61,915 +12,SBE,Search,Backward search,1162.0,1119.0,1203.0,8.26,48.1,28.9,,0.131,0.551,2744.2,48.0,61,915 +13,LOCO,Search,Leave-one-out,1159.0,1121.0,1200.0,8.23,48.3,27.6,,0.173,0.574,272.5,8.4,61,915 +14,Random,Baseline,Baseline,1000.0,960.0,1037.0,10.59,31.5,36.8,,0.001,0.5,0.0,0.0,61,915 +15,LaplacianScore,Non-search,Distance,942.0,875.0,997.0,11.76,23.1,42.4,,0.639,0.8,0.2,0.0,61,915 diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_0/elo_explorer.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_0/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..f9255f42e8bdeeed026c3bc14eb4240df8da5462 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_0/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 1
Hover or tap a method for its values.
900100011001200130014001500EloLaplacianScoreRandomSBELOCO(R)ReliefFSFSLassoGainRatiomRMRElasticNetMIF-TestCARTMarkovBlanketRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_0/leaderboard_table.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_0/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..d23195efe6376b1141884dd0c1ced2eada86fa89 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_0/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 1
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1364+55 / −486.1721.90.6130.7990.20.0
🥈2MarkovBlanketBackward searchsearch1337+58 / −546.3222.90.4160.4270.20.0
🥉3CARTTreenon-search1321+56 / −486.6726.50.4620.7180.10.0
4F-TestCorrelationnon-search1306+47 / −457.1227.10.7110.9500.00.0
5MIInformation theorynon-search1301+45 / −476.9726.10.6290.8840.40.0
6ElasticNetRegularizationnon-search1288+53 / −537.0227.50.5070.91118.80.0
7mRMRInformation theorynon-search1284+50 / −457.3527.40.5290.7940.30.0
8GainRatioInformation theorynon-search1266+54 / −527.9329.30.6860.9590.10.0
9LassoRegularizationnon-search1256+55 / −527.5429.80.4040.8854.30.0
10SFSForward searchsearch1246+51 / −527.6028.90.3690.7431,022.028.4
11(R)ReliefFDistancenon-search1225+56 / −548.4031.80.6430.86212.90.0
12LOCOLeave-one-outsearch1214+51 / −518.3033.40.2970.662270.30.0
13SBEBackward searchsearch1203+55 / −598.5834.80.1930.6233,600.251.4
14RandomBaseline1000+50 / −5811.5347.8-0.0020.4960.00.0
15LaplacianScoreDistancenon-search960+72 / −10312.4952.60.5720.8500.20.0
diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_0/pareto.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_0/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..132d38e9ecd643f3351248062f61d0d4f4dd0e28 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_0/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,35.2255,24.914,61,True +MI,33.0844,46.8303,61,False +MarkovBlanket,32.2725,22.9065,61,True +F-Test,32.1887,0.9617,61,True +CART,31.1921,16.6514,61,False +mRMR,30.3424,9.7809,61,False +ElasticNet,27.7708,3363.7013,61,False +GainRatio,26.893,4.4998,61,False +(R)ReliefF,26.4397,27478.2019,61,False +SFS,25.8664,16558.7051,61,False +Lasso,25.3324,3382.5705,61,False +LOCO,17.7856,5814.5311,61,False +SBE,15.2593,33760.6033,61,False +Random,0.0,0.0032,61,True +LaplacianScore,-19.8723,4.7223,61,False diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_0/pareto.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..36cbe2a1ffc86e47f1b3132ee6858c3957808b79 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 1
Hover or tap a method for its values.
10-210-110010110210310405101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketF-TestCARTmRMRElasticNetGainRatio(R)ReliefFSFSLassoLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_0/website_leaderboard.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_0/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..1eacb14aeb84f8d97b1a722c78e07ef194c9fab2 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_0/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1364.0,1316.0,1419.0,6.17,63.1,21.9,,0.613,0.799,0.2,0.0,61,183 +2,MarkovBlanket,Search,Backward search,1337.0,1283.0,1395.0,6.32,62.0,22.9,,0.416,0.427,0.2,0.0,61,183 +3,CART,Non-search,Tree,1321.0,1273.0,1377.0,6.67,59.5,26.5,,0.462,0.718,0.1,0.0,61,183 +4,F-Test,Non-search,Correlation,1306.0,1261.0,1353.0,7.12,56.3,27.1,,0.711,0.95,0.0,0.0,61,183 +5,MI,Non-search,Information theory,1301.0,1254.0,1346.0,6.97,57.3,26.1,,0.629,0.884,0.4,0.0,61,183 +6,ElasticNet,Non-search,Regularization,1288.0,1235.0,1341.0,7.02,57.0,27.5,,0.507,0.911,18.8,0.0,61,183 +7,mRMR,Non-search,Information theory,1284.0,1239.0,1334.0,7.35,54.6,27.4,,0.529,0.794,0.3,0.0,61,183 +8,GainRatio,Non-search,Information theory,1266.0,1214.0,1320.0,7.93,50.5,29.3,,0.686,0.959,0.1,0.0,61,183 +9,Lasso,Non-search,Regularization,1256.0,1204.0,1311.0,7.54,53.3,29.8,,0.404,0.885,4.3,0.0,61,183 +10,SFS,Search,Forward search,1246.0,1194.0,1297.0,7.6,52.9,28.9,,0.369,0.743,1022.0,28.4,61,183 +11,(R)ReliefF,Non-search,Distance,1225.0,1171.0,1281.0,8.4,47.2,31.8,,0.643,0.862,12.9,0.0,61,183 +12,LOCO,Search,Leave-one-out,1214.0,1163.0,1265.0,8.3,47.9,33.4,,0.297,0.662,270.3,0.0,61,183 +13,SBE,Search,Backward search,1203.0,1144.0,1258.0,8.58,45.9,34.8,,0.193,0.623,3600.2,51.4,61,183 +14,Random,Baseline,Baseline,1000.0,942.0,1050.0,11.53,24.8,47.8,,-0.002,0.496,0.0,0.0,61,183 +15,LaplacianScore,Non-search,Distance,960.0,857.0,1032.0,12.49,17.9,52.6,,0.572,0.85,0.2,0.0,61,183 diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_1/elo_explorer.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_1/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..1ad5e9b41a6f89e0341c614b19993f9e67e5b84b --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_1/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 2
Hover or tap a method for its values.
900100011001200130014001500EloRandomLaplacianScoreSBELOCOSFSGainRatio(R)ReliefFmRMRLassoF-TestElasticNetMarkovBlanketCARTMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_1/leaderboard_table.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_1/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..920e45f2a75fab23403d689aabb30be7a612280c --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_1/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 2
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1376+55 / −506.2619.10.6430.7330.20.0
🥈2MIInformation theorynon-search1346+48 / −466.8323.70.6520.8480.40.0
🥉3CARTTreenon-search1336+53 / −496.6324.80.4410.6210.10.0
4MarkovBlanketBackward searchsearch1325+49 / −466.8026.40.4040.4320.20.0
5F-TestCorrelationnon-search1318+52 / −467.1927.80.7260.9310.00.0
6ElasticNetRegularizationnon-search1318+50 / −506.8226.10.5030.87019.00.0
7LassoRegularizationnon-search1299+58 / −547.2423.70.3920.8244.20.0
8mRMRInformation theorynon-search1298+52 / −447.6827.00.5840.7190.50.0
9(R)ReliefFDistancenon-search1283+50 / −478.0028.10.6520.79312.70.0
10GainRatioInformation theorynon-search1277+46 / −467.8430.10.6870.9290.10.0
11SFSForward searchsearch1246+52 / −588.0329.20.2810.6561,774.843.7
12LOCOLeave-one-outsearch1224+55 / −568.3631.50.2180.601272.70.0
13SBEBackward searchsearch1222+60 / −638.5433.00.1720.5533,600.150.3
14LaplacianScoreDistancenon-search1012+73 / −10612.1849.80.6340.8510.20.0
15RandomBaseline1000+58 / −7111.5945.10.0000.4960.00.0
diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_1/pareto.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_1/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..1534d4a1b850951b2c59e79d07bc2a9922f67583 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_1/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,34.3064,25.0681,61,True +MI,32.6081,46.7539,61,False +CART,31.8131,16.7132,61,True +MarkovBlanket,29.6455,22.7641,61,False +mRMR,29.2071,17.1173,61,False +F-Test,29.0418,0.9185,61,True +ElasticNet,27.3953,3341.7491,61,False +GainRatio,26.93,4.5095,61,False +(R)ReliefF,26.498,27555.5181,61,False +Lasso,26.1224,3404.0402,61,False +SFS,23.9998,24500.2443,61,False +LOCO,18.4367,5842.6711,61,False +SBE,16.1755,31655.945,61,False +Random,0.0,0.0028,61,True +LaplacianScore,-33.5532,4.5705,61,False diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_1/pareto.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..90c578528a48bae3ccd33e7f788ceff942c26f09 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 2
Hover or tap a method for its values.
10-210-110010110210310405101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICARTMarkovBlanketmRMRF-TestElasticNetGainRatio(R)ReliefFLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_1/website_leaderboard.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_1/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..d853926611e1a9b58c5e6387465a6f216751aa0f --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_1/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1376.0,1326.0,1431.0,6.26,62.4,19.1,,0.643,0.733,0.2,0.0,61,183 +2,MI,Non-search,Information theory,1346.0,1300.0,1394.0,6.83,58.4,23.7,,0.652,0.848,0.4,0.0,61,183 +3,CART,Non-search,Tree,1336.0,1287.0,1389.0,6.63,59.8,24.8,,0.441,0.621,0.1,0.0,61,183 +4,MarkovBlanket,Search,Backward search,1325.0,1279.0,1374.0,6.8,58.6,26.4,,0.404,0.432,0.2,0.0,61,183 +5,F-Test,Non-search,Correlation,1318.0,1272.0,1370.0,7.19,55.8,27.8,,0.726,0.931,0.0,0.0,61,183 +6,ElasticNet,Non-search,Regularization,1318.0,1268.0,1368.0,6.82,58.4,26.1,,0.503,0.87,19.0,0.0,61,183 +7,Lasso,Non-search,Regularization,1299.0,1245.0,1357.0,7.24,55.4,23.7,,0.392,0.824,4.2,0.0,61,183 +8,mRMR,Non-search,Information theory,1298.0,1254.0,1350.0,7.68,52.3,27.0,,0.584,0.719,0.5,0.0,61,183 +9,(R)ReliefF,Non-search,Distance,1283.0,1236.0,1333.0,8.0,50.0,28.1,,0.652,0.793,12.7,0.0,61,183 +10,GainRatio,Non-search,Information theory,1277.0,1231.0,1323.0,7.84,51.1,30.1,,0.687,0.929,0.1,0.0,61,183 +11,SFS,Search,Forward search,1246.0,1188.0,1298.0,8.03,49.8,29.2,,0.281,0.656,1774.8,43.7,61,183 +12,LOCO,Search,Leave-one-out,1224.0,1168.0,1279.0,8.36,47.4,31.5,,0.218,0.601,272.7,0.0,61,183 +13,SBE,Search,Backward search,1222.0,1159.0,1282.0,8.54,46.2,33.0,,0.172,0.553,3600.1,50.3,61,183 +14,LaplacianScore,Non-search,Distance,1012.0,906.0,1085.0,12.18,20.1,49.8,,0.634,0.851,0.2,0.0,61,183 +15,Random,Baseline,Baseline,1000.0,929.0,1058.0,11.59,24.4,45.1,,0.0,0.496,0.0,0.0,61,183 diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_2/elo_explorer.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_2/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..e5bc0408ca08f00e26db8dd711b89d61197f67f5 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_2/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 3
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandomLOCOSFSGainRatiomRMRLassoSBE(R)ReliefFF-TestElasticNetMarkovBlanketCARTMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_2/leaderboard_table.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_2/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..58b22648d1ad7fd051c74becee89d4ef47e51e30 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_2/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 3
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1311+51 / −436.2818.00.6760.6890.20.0
🥈2MIInformation theorynon-search1288+54 / −516.5821.30.6490.8020.40.0
🥉3CARTTreenon-search1281+52 / −546.5119.90.4340.5590.10.0
4ElasticNetRegularizationnon-search1253+59 / −556.8521.90.4700.76918.90.0
5MarkovBlanketBackward searchsearch1253+50 / −497.2523.90.4170.4160.20.0
6F-TestCorrelationnon-search1242+46 / −437.2323.80.6970.8790.00.0
7(R)ReliefFDistancenon-search1215+56 / −507.7022.70.6680.74512.70.0
8SBEBackward searchsearch1208+53 / −527.7526.50.1420.5473,307.848.1
9LassoRegularizationnon-search1206+57 / −537.3923.20.3740.7234.30.0
10mRMRInformation theorynon-search1191+47 / −468.1924.40.6240.6690.60.0
11GainRatioInformation theorynon-search1183+46 / −438.3826.80.6850.8710.10.0
12SFSForward searchsearch1163+47 / −498.2727.40.2360.6242,832.346.4
13LOCOLeave-one-outsearch1146+57 / −578.5128.20.1500.560255.98.7
14RandomBaseline1000+54 / −6510.7738.1-0.0000.5170.00.0
15LaplacianScoreDistancenon-search905+76 / −10712.3445.60.6450.8240.20.0
diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_2/pareto.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_2/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..0db6c04e531549dd6fe558b4e30091dc761113b1 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_2/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,28.8386,25.1686,61,True +CART,24.7201,16.8307,61,True +MI,22.8373,47.0609,61,False +(R)ReliefF,21.9241,27516.8811,61,False +MarkovBlanket,21.0145,22.5936,61,False +F-Test,20.5771,0.9392,61,True +mRMR,20.4011,35.1491,61,False +ElasticNet,19.7824,3207.7398,61,False +GainRatio,18.1693,4.5521,61,False +Lasso,16.1951,3382.7617,61,False +SBE,14.2495,28904.5926,61,False +SFS,13.5981,30481.3776,61,False +LOCO,11.8517,5885.0869,61,False +Random,0.0,0.0138,61,True +LaplacianScore,-243.5934,4.929,61,False diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_2/pareto.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..89f445364370d77649f246382d87333ec9489e9c --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 3
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMI(R)ReliefFMarkovBlanketF-TestmRMRElasticNetGainRatioLassoSBESFSLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_2/website_leaderboard.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_2/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..1231c1f139616b8e84017ac0d2dcf62257798ba4 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_2/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1311.0,1268.0,1362.0,6.28,62.3,18.0,,0.676,0.689,0.2,0.0,61,183 +2,MI,Non-search,Information theory,1288.0,1237.0,1342.0,6.58,60.1,21.3,,0.649,0.802,0.4,0.0,61,183 +3,CART,Non-search,Tree,1281.0,1227.0,1333.0,6.51,60.6,19.9,,0.434,0.559,0.1,0.0,61,183 +4,ElasticNet,Non-search,Regularization,1253.0,1198.0,1312.0,6.85,58.2,21.9,,0.47,0.769,18.9,0.0,61,183 +5,MarkovBlanket,Search,Backward search,1253.0,1204.0,1303.0,7.25,55.4,23.9,,0.417,0.416,0.2,0.0,61,183 +6,F-Test,Non-search,Correlation,1242.0,1199.0,1288.0,7.23,55.5,23.8,,0.697,0.879,0.0,0.0,61,183 +7,(R)ReliefF,Non-search,Distance,1215.0,1165.0,1271.0,7.7,52.2,22.7,,0.668,0.745,12.7,0.0,61,183 +8,SBE,Search,Backward search,1208.0,1156.0,1261.0,7.75,51.8,26.5,,0.142,0.547,3307.8,48.1,61,183 +9,Lasso,Non-search,Regularization,1206.0,1153.0,1263.0,7.39,54.4,23.2,,0.374,0.723,4.3,0.0,61,183 +10,mRMR,Non-search,Information theory,1191.0,1145.0,1238.0,8.19,48.7,24.4,,0.624,0.669,0.6,0.0,61,183 +11,GainRatio,Non-search,Information theory,1183.0,1140.0,1229.0,8.38,47.3,26.8,,0.685,0.871,0.1,0.0,61,183 +12,SFS,Search,Forward search,1163.0,1114.0,1210.0,8.27,48.1,27.4,,0.236,0.624,2832.3,46.4,61,183 +13,LOCO,Search,Leave-one-out,1146.0,1089.0,1203.0,8.51,46.4,28.2,,0.15,0.56,255.9,8.7,61,183 +14,Random,Baseline,Baseline,1000.0,935.0,1054.0,10.77,30.2,38.1,,-0.0,0.517,0.0,0.0,61,183 +15,LaplacianScore,Non-search,Distance,905.0,798.0,981.0,12.34,19.0,45.6,,0.645,0.824,0.2,0.0,61,183 diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_3/elo_explorer.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_3/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..ccc53bc229144b5a9efcd79f9e857986fadaae35 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_3/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 4
Hover or tap a method for its values.
80090010001100120013001400EloLaplacianScoreRandomLOCOSFSmRMRGainRatioLasso(R)ReliefFElasticNetMarkovBlanketSBEF-TestMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_3/leaderboard_table.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_3/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..08521fd19e49fcfdcad910b49c9a6c778939e0ad --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_3/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 4
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1334+62 / −515.5512.40.6510.5990.20.0
🥈2CARTTreenon-search1250+47 / −466.7017.80.3890.5020.10.0
🥉3MIInformation theorynon-search1237+56 / −516.7919.70.6280.7160.30.0
4F-TestCorrelationnon-search1224+48 / −467.1017.10.6610.7810.00.0
5SBEBackward searchsearch1209+58 / −547.3721.30.0960.5212,396.945.4
6MarkovBlanketBackward searchsearch1194+48 / −517.8421.90.4360.4510.20.0
7ElasticNetRegularizationnon-search1187+57 / −527.6520.20.4100.65719.00.0
8(R)ReliefFDistancenon-search1173+50 / −507.8718.90.6460.65712.90.0
9LassoRegularizationnon-search1165+53 / −467.7620.50.3220.6344.30.0
10GainRatioInformation theorynon-search1153+41 / −468.3321.90.6520.7660.10.0
11mRMRInformation theorynon-search1145+51 / −518.4919.30.6710.6000.78.2
12SFSForward searchsearch1132+49 / −558.2322.70.2260.5633,600.050.8
13LOCOLeave-one-outsearch1109+55 / −518.4023.50.1170.531262.014.2
14RandomBaseline1000+60 / −5810.0429.10.0020.4900.00.0
15LaplacianScoreDistancenon-search930+72 / −9411.8737.20.6790.7630.20.0
diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_3/pareto.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_3/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..8623d68792ff43b1705ef0a0e9a20403ea0c9681 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_3/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,22.1956,24.8874,61,True +CART,15.5229,16.6247,61,True +mRMR,12.0662,74.8179,61,False +F-Test,12.0424,0.9472,61,True +(R)ReliefF,9.8279,27549.7605,61,False +MarkovBlanket,9.7696,22.5018,61,False +MI,7.9793,46.8719,61,False +GainRatio,7.6192,4.497,61,False +SBE,7.3576,23467.1313,61,False +ElasticNet,6.0027,3217.8078,61,False +SFS,5.9918,33201.539,61,False +LOCO,5.9178,5932.2624,61,False +Lasso,5.4475,3399.8528,61,False +Random,0.0,0.0027,61,True +LaplacianScore,-45573766721072.28,4.6733,61,False diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_3/pareto.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..017dc17a37c8ffc08eaa66c78f9e334642484bc2 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 4
Hover or tap a method for its values.
10-210-110010110210310405101520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTmRMRF-Test(R)ReliefFMarkovBlanketMIGainRatioSBEElasticNetSFSLOCOLassoRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_3/website_leaderboard.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_3/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..6023df03c2a1132e3906bf80877b9d5a6a7984e4 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_3/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1334.0,1283.0,1396.0,5.55,67.5,12.4,,0.651,0.599,0.2,0.0,61,183 +2,CART,Non-search,Tree,1250.0,1204.0,1297.0,6.7,59.2,17.8,,0.389,0.502,0.1,0.0,61,183 +3,MI,Non-search,Information theory,1237.0,1186.0,1293.0,6.79,58.6,19.7,,0.628,0.716,0.3,0.0,61,183 +4,F-Test,Non-search,Correlation,1224.0,1178.0,1272.0,7.1,56.4,17.1,,0.661,0.781,0.0,0.0,61,183 +5,SBE,Search,Backward search,1209.0,1155.0,1267.0,7.37,54.5,21.3,,0.096,0.521,2396.9,45.4,61,183 +6,MarkovBlanket,Search,Backward search,1194.0,1143.0,1242.0,7.84,51.1,21.9,,0.436,0.451,0.2,0.0,61,183 +7,ElasticNet,Non-search,Regularization,1187.0,1135.0,1244.0,7.65,52.5,20.2,,0.41,0.657,19.0,0.0,61,183 +8,(R)ReliefF,Non-search,Distance,1173.0,1123.0,1223.0,7.87,50.9,18.9,,0.646,0.657,12.9,0.0,61,183 +9,Lasso,Non-search,Regularization,1165.0,1119.0,1218.0,7.76,51.7,20.5,,0.322,0.634,4.3,0.0,61,183 +10,GainRatio,Non-search,Information theory,1153.0,1107.0,1194.0,8.33,47.6,21.9,,0.652,0.766,0.1,0.0,61,183 +11,mRMR,Non-search,Information theory,1145.0,1094.0,1196.0,8.49,46.5,19.3,,0.671,0.6,0.7,8.2,61,183 +12,SFS,Search,Forward search,1132.0,1077.0,1181.0,8.23,48.3,22.7,,0.226,0.563,3600.0,50.8,61,183 +13,LOCO,Search,Leave-one-out,1109.0,1058.0,1164.0,8.4,47.1,23.5,,0.117,0.531,262.0,14.2,61,183 +14,Random,Baseline,Baseline,1000.0,942.0,1060.0,10.04,35.5,29.1,,0.002,0.49,0.0,0.0,61,183 +15,LaplacianScore,Non-search,Distance,930.0,836.0,1002.0,11.87,22.4,37.2,,0.679,0.763,0.2,0.0,61,183 diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_4/elo_explorer.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_4/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..744330f95196441145a12eb5ff7c35d39301c093 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_4/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 5
Hover or tap a method for its values.
90010001100120013001400EloLaplacianScoreRandomLOCOGainRatioMarkovBlanketSFSmRMRLassoSBEMIElasticNet(R)ReliefFCARTF-TestRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_4/leaderboard_table.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_4/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..fef8dbe690af2afd268ac45aa3e5d6a34dc85d82 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_4/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 5
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1309+49 / −445.6310.50.5820.5110.20.0
🥈2F-TestCorrelationnon-search1205+60 / −497.3612.00.5640.6890.00.0
🥉3CARTTreenon-search1188+50 / −497.0014.20.2980.4840.10.0
4(R)ReliefFDistancenon-search1178+52 / −497.6714.40.6050.58812.70.0
5ElasticNetRegularizationnon-search1170+53 / −487.4513.20.2870.56118.80.0
6MIInformation theorynon-search1169+53 / −557.4615.40.5620.6240.40.0
7SBEBackward searchsearch1138+50 / −537.9016.70.0590.5131,030.442.3
8LassoRegularizationnon-search1124+51 / −517.9914.80.2220.5574.30.0
9mRMRInformation theorynon-search1118+47 / −508.2315.20.6630.5320.811.5
10SFSForward searchsearch1112+49 / −557.8216.20.1860.5233,600.050.0
11MarkovBlanketBackward searchsearch1110+51 / −528.4017.10.4000.4500.20.0
12GainRatioInformation theorynon-search1106+45 / −498.4416.80.5560.6730.10.0
13LOCOLeave-one-outsearch1094+47 / −478.3317.00.0840.514263.518.6
14RandomBaseline1000+55 / −579.5420.90.0020.5010.00.0
15LaplacianScoreDistancenon-search961+59 / −7610.7127.30.6670.7110.20.0
diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_4/pareto.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_4/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..d1dada4d7b9a17c86c1f57fe0d33b13000cfcba6 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_4/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,13.6467,25.3211,61,True +F-Test,12.0336,1.5004,61,True +(R)ReliefF,9.145,27571.5035,61,False +CART,8.6848,16.8842,61,False +ElasticNet,7.8987,3290.8737,61,False +GainRatio,7.3517,4.5094,61,False +Lasso,6.5855,3386.7504,61,False +LOCO,5.9766,5932.9496,61,False +MarkovBlanket,5.5685,21.6681,61,False +mRMR,4.3016,133.9518,61,False +SBE,3.6805,15416.9435,61,False +SFS,2.0048,35097.8319,61,False +Random,0.0,0.0391,61,True +MI,-9114753344196.406,46.7153,61,False +LaplacianScore,-108788991527598.16,4.6489,61,False diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_4/pareto.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..18817209d5ba521fa7579ced909f03f6dd4cacd4 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, TabICLv2, budget 5
Hover or tap a method for its values.
10-1100101102103104-8e+12-6e+12-4e+12-2e+120Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceF-Test(R)ReliefFCARTElasticNetGainRatioLassoLOCOMarkovBlanketmRMRSBESFSRandomMIDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_4/website_leaderboard.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_4/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..75621e30f167e82bf1c6d04f85bab3e47c366122 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_4/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1309.0,1265.0,1358.0,5.63,66.9,10.5,,0.582,0.511,0.2,0.0,61,183 +2,F-Test,Non-search,Correlation,1205.0,1156.0,1265.0,7.36,54.6,12.0,,0.564,0.689,0.0,0.0,61,183 +3,CART,Non-search,Tree,1188.0,1139.0,1238.0,7.0,57.1,14.2,,0.298,0.484,0.1,0.0,61,183 +4,(R)ReliefF,Non-search,Distance,1178.0,1129.0,1230.0,7.67,52.3,14.4,,0.605,0.588,12.7,0.0,61,183 +5,ElasticNet,Non-search,Regularization,1170.0,1122.0,1223.0,7.45,53.9,13.2,,0.287,0.561,18.8,0.0,61,183 +6,MI,Non-search,Information theory,1169.0,1114.0,1222.0,7.46,53.8,15.4,,0.562,0.624,0.4,0.0,61,183 +7,SBE,Search,Backward search,1138.0,1085.0,1188.0,7.9,50.7,16.7,,0.059,0.513,1030.4,42.3,61,182 +8,Lasso,Non-search,Regularization,1124.0,1073.0,1175.0,7.99,50.1,14.8,,0.222,0.557,4.3,0.0,61,183 +9,mRMR,Non-search,Information theory,1118.0,1068.0,1165.0,8.23,48.3,15.2,,0.663,0.532,0.8,11.5,61,183 +10,SFS,Search,Forward search,1112.0,1057.0,1161.0,7.82,51.3,16.2,,0.186,0.523,3600.0,50.0,61,182 +11,MarkovBlanket,Search,Backward search,1110.0,1058.0,1161.0,8.4,47.1,17.1,,0.4,0.45,0.2,0.0,61,183 +12,GainRatio,Non-search,Information theory,1106.0,1057.0,1151.0,8.44,46.8,16.8,,0.556,0.673,0.1,0.0,61,183 +13,LOCO,Search,Leave-one-out,1094.0,1047.0,1141.0,8.33,47.6,17.0,,0.084,0.514,263.5,18.6,61,183 +14,Random,Baseline,Baseline,1000.0,943.0,1055.0,9.54,39.0,20.9,,0.002,0.501,0.0,0.0,61,183 +15,LaplacianScore,Non-search,Distance,961.0,885.0,1020.0,10.71,30.6,27.3,,0.667,0.711,0.2,0.0,61,183 diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_all/elo_explorer.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_all/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..a4dd8424a1a3e2c2ea098b0bd6c9d38008cc143d --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_all/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: classification, TabICLv2, all budgets
Hover or tap a method for its values.
90010001100120013001400EloLaplacianScoreRandomLOCOSFSGainRatioSBEmRMRLasso(R)ReliefFElasticNetMarkovBlanketF-TestMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_all/leaderboard_table.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_all/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..012de6eb77eaee3fdb925ad2ed267e29378191a7 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_all/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: classification, TabICLv2, all budgets
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1333+36 / −305.9816.40.6330.6660.20.0
🥈2CARTTreenon-search1270+42 / −396.7020.60.4050.5770.10.0
🥉3MIInformation theorynon-search1263+35 / −356.9321.20.6240.7750.40.0
4F-TestCorrelationnon-search1254+33 / −327.2021.60.6720.8460.00.0
5ElasticNetRegularizationnon-search1238+42 / −407.1621.80.4350.75418.90.0
6MarkovBlanketBackward searchsearch1238+41 / −397.3222.40.4150.4350.20.0
7(R)ReliefFDistancenon-search1210+40 / −357.9323.20.6430.72912.70.0
8LassoRegularizationnon-search1205+41 / −367.5822.40.3430.7254.30.0
9mRMRInformation theorynon-search1203+34 / −337.9922.70.6120.6630.63.9
10GainRatioInformation theorynon-search1192+35 / −358.1925.00.6530.8400.10.0
11SBEBackward searchsearch1192+42 / −458.0326.50.1310.5512,739.547.5
12SFSForward searchsearch1175+38 / −417.9924.90.2690.6222,337.943.9
13LOCOLeave-one-outsearch1153+41 / −438.3826.70.1730.574262.08.3
14RandomBaseline1000+36 / −3810.6936.20.0010.5000.00.0
15LaplacianScoreDistancenon-search954+57 / −7811.9242.50.6390.8000.20.0
diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_all/pareto.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_all/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..4388b403056fce97f81846c95b5c0846d5b2dffe --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_all/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,32.0406,25.0718,61,True +CART,28.4457,16.7408,61,True +MI,28.151,46.8464,61,False +F-Test,26.8605,1.0534,61,True +mRMR,26.4193,54.1634,61,False +MarkovBlanket,26.2547,22.4868,61,False +(R)ReliefF,24.0087,27534.373,61,False +ElasticNet,23.91,3284.3743,61,False +GainRatio,23.1566,4.5136,61,False +Lasso,22.2361,3391.1951,61,False +SFS,21.301,27967.9477,61,False +LOCO,15.8981,5881.5002,61,False +SBE,14.2007,26641.0473,61,False +Random,0.0,0.0123,61,True +LaplacianScore,-22.8035,4.7088,61,False diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_all/pareto.html b/data/subsets/tasks_classification/models_tabiclv2/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..0dd659cda870b6db144e37d8c8aa89f2b4484829 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: classification, TabICLv2, all budgets
Hover or tap a method for its values.
10-210-1100101102103104051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIF-TestmRMRMarkovBlanket(R)ReliefFElasticNetGainRatioLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_classification/models_tabiclv2/budgets_all/website_leaderboard.csv b/data/subsets/tasks_classification/models_tabiclv2/budgets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..3da2854daedb0289813bf8ec8b69f1b701a31010 --- /dev/null +++ b/data/subsets/tasks_classification/models_tabiclv2/budgets_all/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1333.0,1303.0,1369.0,5.98,64.4,16.4,,0.633,0.666,0.2,0.0,61,915 +2,CART,Non-search,Tree,1270.0,1231.0,1312.0,6.7,59.3,20.6,,0.405,0.577,0.1,0.0,61,915 +3,MI,Non-search,Information theory,1263.0,1228.0,1298.0,6.93,57.6,21.2,,0.624,0.775,0.4,0.0,61,915 +4,F-Test,Non-search,Correlation,1254.0,1222.0,1287.0,7.2,55.7,21.6,,0.672,0.846,0.0,0.0,61,915 +5,ElasticNet,Non-search,Regularization,1238.0,1198.0,1280.0,7.16,56.0,21.8,,0.435,0.754,18.9,0.0,61,915 +6,MarkovBlanket,Search,Backward search,1238.0,1199.0,1279.0,7.32,54.8,22.4,,0.415,0.435,0.2,0.0,61,915 +7,(R)ReliefF,Non-search,Distance,1210.0,1175.0,1250.0,7.93,50.5,23.2,,0.643,0.729,12.7,0.0,61,915 +8,Lasso,Non-search,Regularization,1205.0,1169.0,1246.0,7.58,53.0,22.4,,0.343,0.725,4.3,0.0,61,915 +9,mRMR,Non-search,Information theory,1203.0,1170.0,1237.0,7.99,50.1,22.7,,0.612,0.663,0.6,3.9,61,915 +10,GainRatio,Non-search,Information theory,1192.0,1157.0,1227.0,8.19,48.7,25.0,,0.653,0.84,0.1,0.0,61,915 +11,SBE,Search,Backward search,1192.0,1147.0,1234.0,8.03,49.8,26.5,,0.131,0.551,2739.5,47.5,61,914 +12,SFS,Search,Forward search,1175.0,1134.0,1213.0,7.99,50.1,24.9,,0.269,0.622,2337.9,43.9,61,914 +13,LOCO,Search,Leave-one-out,1153.0,1110.0,1194.0,8.38,47.3,26.7,,0.173,0.574,262.0,8.3,61,915 +14,Random,Baseline,Baseline,1000.0,962.0,1036.0,10.69,30.8,36.2,,0.001,0.5,0.0,0.0,61,915 +15,LaplacianScore,Non-search,Distance,954.0,876.0,1011.0,11.92,22.0,42.5,,0.639,0.8,0.2,0.0,61,915 diff --git a/data/subsets/tasks_multiclass/models_all/budgets_0/pareto.html b/data/subsets/tasks_multiclass/models_all/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..abf305c07ed38e6ebf0d49598811518ca78298f4 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, all models, budget 1
Hover or tap a method for its values.
10-210-110010110210310410505101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMarkovBlanketCARTMI(R)ReliefFGainRatioElasticNetmRMRF-TestLassoSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_all/budgets_0/per_model.csv b/data/subsets/tasks_multiclass/models_all/budgets_0/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..4fab719d53b1a24ad23abca4b3e687d7af549611 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_0/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,CART,Non-search,Tree,1516.0,1649.0,1,1688.0,1,1323.0,3,1504.0,2,1543.0,1 +2,MarkovBlanket,Search,Backward search,1490.0,1614.0,3,1562.0,3,1355.0,1,1509.0,1,1520.0,2 +3,RFImportance,Non-search,Tree,1468.0,1636.0,2,1581.0,2,1312.0,4,1427.0,4,1499.0,3 +4,MI,Non-search,Information theory,1423.0,1486.0,4,1521.0,4,1308.0,5,1443.0,3,1460.0,4 +5,ElasticNet,Non-search,Regularization,1353.0,1481.0,5,1460.0,5,1182.0,9,1353.0,6,1391.0,5 +6,F-Test,Non-search,Correlation,1348.0,1423.0,10,1397.0,9,1258.0,7,1372.0,5,1382.0,7 +7,mRMR,Non-search,Information theory,1332.0,1437.0,7,1450.0,6,1157.0,11,1332.0,7,1387.0,6 +8,GainRatio,Non-search,Information theory,1331.0,1396.0,11,1382.0,10,1326.0,2,1320.0,8,1320.0,9 +9,(R)ReliefF,Non-search,Distance,1319.0,1423.0,9,1397.0,8,1262.0,6,1283.0,10,1319.0,10 +10,Lasso,Non-search,Regularization,1308.0,1439.0,6,1408.0,7,1170.0,10,1284.0,9,1337.0,8 +11,SFS,Search,Forward search,1277.0,1430.0,8,1330.0,11,1201.0,8,1251.0,11,1259.0,13 +12,SBE,Search,Backward search,1241.0,1362.0,13,1279.0,13,1131.0,12,1232.0,12,1286.0,11 +13,LOCO,Search,Leave-one-out,1234.0,1382.0,12,1305.0,12,1087.0,14,1214.0,13,1269.0,12 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,15,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,1000.0,977.0,15,947.0,15,1091.0,13,960.0,15,946.0,15 diff --git a/data/subsets/tasks_multiclass/models_all/budgets_0/per_model_table.html b/data/subsets/tasks_multiclass/models_all/budgets_0/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..6cf28004f76ebf9029c2893609a9b2fef9ccd3aa --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_0/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, all models, budget 1 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1CART1516🥇1649🥇1688🥉1323🥈1504🥇1543
🥈2MarkovBlanket1490🥉1614🥉1562🥇1355🥇1509🥈1520
🥉3RFImportance1468🥈1636🥈158113121427🥉1499
4MI1423148615211308🥉14431460
5ElasticNet135314811460118213531391
6F-Test134814231397125813721382
7mRMR133214371450115713321387
8GainRatio133113961382🥈132613201320
9(R)ReliefF131914231397126212831319
10Lasso130814391408117012841337
11SFS127714301330120112511259
12SBE124113621279113112321286
13LOCO123413821305108712141269
14Random100010001000100010001000
15LaplacianScore10009779471091960946
diff --git a/data/subsets/tasks_multiclass/models_all/budgets_1/pareto.html b/data/subsets/tasks_multiclass/models_all/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..93ad2d6b80a961e3c76ca9c4fec442d7a6faed05 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, all models, budget 2
Hover or tap a method for its values.
10-310-210-110010110210310410505101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketCARTGainRatioElasticNet(R)ReliefFF-TestmRMRLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_all/budgets_1/per_model.csv b/data/subsets/tasks_multiclass/models_all/budgets_1/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..ea53986e0e5cb2a61d157eea85d06ee670ba7d67 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_1/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1578.0,1732.0,1,1587.0,2,1505.0,1,1516.0,1,1631.0,1 +2,CART,Non-search,Tree,1524.0,1602.0,2,1647.0,1,1380.0,5,1511.0,2,1568.0,2 +3,MI,Non-search,Information theory,1481.0,1522.0,4,1554.0,3,1402.0,2,1450.0,3,1547.0,3 +4,MarkovBlanket,Search,Backward search,1475.0,1566.0,3,1513.0,4,1398.0,4,1445.0,4,1522.0,4 +5,ElasticNet,Non-search,Regularization,1429.0,1517.0,5,1508.0,5,1328.0,6,1402.0,6,1458.0,6 +6,Lasso,Non-search,Regularization,1407.0,1483.0,6,1398.0,9,1402.0,3,1357.0,7,1453.0,7 +7,(R)ReliefF,Non-search,Distance,1386.0,1455.0,7,1416.0,7,1286.0,9,1412.0,5,1420.0,8 +8,mRMR,Non-search,Information theory,1378.0,1453.0,8,1410.0,8,1294.0,8,1330.0,9,1463.0,5 +9,F-Test,Non-search,Correlation,1368.0,1432.0,9,1416.0,6,1282.0,10,1352.0,8,1415.0,9 +10,GainRatio,Non-search,Information theory,1339.0,1382.0,10,1390.0,10,1307.0,7,1312.0,10,1354.0,10 +11,LOCO,Search,Leave-one-out,1253.0,1355.0,12,1244.0,12,1221.0,12,1218.0,11,1270.0,11 +12,SFS,Search,Forward search,1251.0,1365.0,11,1296.0,11,1185.0,14,1213.0,12,1243.0,12 +13,SBE,Search,Backward search,1222.0,1289.0,13,1211.0,13,1248.0,11,1173.0,13,1216.0,13 +14,LaplacianScore,Non-search,Distance,1080.0,1011.0,14,1035.0,14,1208.0,13,1037.0,14,1044.0,14 +15,Random,Baseline,Baseline,1000.0,1000.0,15,1000.0,15,1000.0,15,1000.0,15,1000.0,15 diff --git a/data/subsets/tasks_multiclass/models_all/budgets_1/per_model_table.html b/data/subsets/tasks_multiclass/models_all/budgets_1/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..1ad58858a6057100ce82af439a32d3e6acc7fe57 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_1/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, all models, budget 2 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1578🥇1732🥈1587🥇1505🥇1516🥇1631
🥈2CART1524🥈1602🥇16471380🥈1511🥈1568
🥉3MI14811522🥉1554🥈1402🥉1450🥉1547
4MarkovBlanket1475🥉15661513139814451522
5ElasticNet142915171508132814021458
6Lasso140714831398🥉140213571453
7(R)ReliefF138614551416128614121420
8mRMR137814531410129413301463
9F-Test136814321416128213521415
10GainRatio133913821390130713121354
11LOCO125313551244122112181270
12SFS125113651296118512131243
13SBE122212891211124811731216
14LaplacianScore108010111035120810371044
15Random100010001000100010001000
diff --git a/data/subsets/tasks_multiclass/models_all/budgets_2/pareto.html b/data/subsets/tasks_multiclass/models_all/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..2e44181ab76b3ac377f04234fbe49ccb0a727fc2 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, all models, budget 3
Hover or tap a method for its values.
10-210-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMImRMRMarkovBlanketGainRatio(R)ReliefFF-TestElasticNetCARTLassoSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_all/budgets_2/per_model.csv b/data/subsets/tasks_multiclass/models_all/budgets_2/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..fa5e4c6070f33922288ad6924585bf37884353c5 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_2/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1412.0,1479.0,1,1342.0,1,1439.0,1,1436.0,1,1402.0,1 +2,MI,Non-search,Information theory,1317.0,1370.0,2,1277.0,3,1266.0,2,1383.0,2,1310.0,2 +3,CART,Non-search,Tree,1273.0,1290.0,6,1306.0,2,1237.0,3,1254.0,6,1296.0,3 +4,ElasticNet,Non-search,Regularization,1251.0,1295.0,5,1254.0,4,1176.0,6,1282.0,4,1263.0,5 +5,MarkovBlanket,Search,Backward search,1236.0,1304.0,4,1196.0,7,1199.0,4,1272.0,5,1226.0,7 +6,F-Test,Non-search,Correlation,1228.0,1254.0,8,1227.0,5,1150.0,9,1320.0,3,1208.0,9 +7,Lasso,Non-search,Regularization,1210.0,1323.0,3,1165.0,9,1176.0,7,1190.0,7,1212.0,8 +8,(R)ReliefF,Non-search,Distance,1203.0,1259.0,7,1152.0,10,1154.0,8,1190.0,8,1277.0,4 +9,mRMR,Non-search,Information theory,1190.0,1212.0,9,1189.0,8,1181.0,5,1141.0,10,1240.0,6 +10,GainRatio,Non-search,Information theory,1156.0,1191.0,10,1198.0,6,1105.0,10,1163.0,9,1132.0,10 +11,SBE,Search,Backward search,1062.0,1090.0,12,1063.0,11,990.0,12,1122.0,11,1051.0,11 +12,LOCO,Search,Leave-one-out,1022.0,1085.0,13,1000.0,13,950.0,13,1056.0,12,1026.0,12 +13,SFS,Search,Forward search,1005.0,1114.0,11,1015.0,12,939.0,14,1011.0,13,944.0,14 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,14,1000.0,11,1000.0,14,1000.0,13 +15,LaplacianScore,Non-search,Distance,879.0,909.0,15,844.0,15,895.0,15,876.0,15,866.0,15 diff --git a/data/subsets/tasks_multiclass/models_all/budgets_2/per_model_table.html b/data/subsets/tasks_multiclass/models_all/budgets_2/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..e17eed29882055d778a00cb45982cb5739805ac6 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_2/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, all models, budget 3 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1412🥇1479🥇1342🥇1439🥇1436🥇1402
🥈2MI1317🥈1370🥉1277🥈1266🥈1383🥈1310
🥉3CART12731290🥈1306🥉12371254🥉1296
4ElasticNet125112951254117612821263
5MarkovBlanket123613041196119912721226
6F-Test1228125412271150🥉13201208
7Lasso1210🥉13231165117611901212
8(R)ReliefF120312591152115411901277
9mRMR119012121189118111411240
10GainRatio115611911198110511631132
11SBE10621090106399011221051
12LOCO10221085100095010561026
13SFS1005111410159391011944
14Random100010001000100010001000
15LaplacianScore879909844895876866
diff --git a/data/subsets/tasks_multiclass/models_all/budgets_3/pareto.html b/data/subsets/tasks_multiclass/models_all/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..150e98a5bb1c7cdfc3af96b69345d71dd666d7a0 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, all models, budget 4
Hover or tap a method for its values.
10-310-210-11001011021031041050510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketmRMRElasticNetCARTF-Test(R)ReliefFGainRatioLassoSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_all/budgets_3/per_model.csv b/data/subsets/tasks_multiclass/models_all/budgets_3/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..1cefa041270d206bbe0a19221906242449268006 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_3/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1482.0,1428.0,1,1534.0,1,1472.0,1,1425.0,1,1635.0,1 +2,MI,Non-search,Information theory,1377.0,1234.0,4,1454.0,2,1439.0,2,1365.0,2,1446.0,2 +3,CART,Non-search,Tree,1307.0,1212.0,6,1343.0,6,1407.0,3,1307.0,3,1306.0,5 +4,ElasticNet,Non-search,Regularization,1291.0,1252.0,3,1343.0,5,1377.0,5,1236.0,6,1287.0,6 +5,MarkovBlanket,Search,Backward search,1289.0,1218.0,5,1366.0,4,1310.0,7,1271.0,4,1314.0,4 +6,Lasso,Non-search,Regularization,1262.0,1275.0,2,1294.0,8,1314.0,6,1214.0,7,1246.0,7 +7,(R)ReliefF,Non-search,Distance,1253.0,1120.0,10,1271.0,10,1392.0,4,1184.0,9,1345.0,3 +8,F-Test,Non-search,Correlation,1238.0,1160.0,8,1334.0,7,1295.0,8,1245.0,5,1187.0,10 +9,GainRatio,Non-search,Information theory,1229.0,1186.0,7,1369.0,3,1216.0,10,1163.0,11,1243.0,8 +10,mRMR,Non-search,Information theory,1203.0,1131.0,9,1278.0,9,1249.0,9,1180.0,10,1205.0,9 +11,SBE,Search,Backward search,1154.0,1090.0,12,1204.0,11,1129.0,11,1184.0,8,1182.0,11 +12,LOCO,Search,Leave-one-out,1106.0,1081.0,13,1161.0,12,1071.0,12,1132.0,12,1092.0,12 +13,SFS,Search,Forward search,1092.0,1094.0,11,1117.0,13,1054.0,13,1132.0,13,1066.0,13 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,15,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,936.0,809.0,15,1038.0,14,946.0,15,876.0,15,1000.0,15 diff --git a/data/subsets/tasks_multiclass/models_all/budgets_3/per_model_table.html b/data/subsets/tasks_multiclass/models_all/budgets_3/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..951a0707ce62172acb90c11df8d26046a54c9551 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_3/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, all models, budget 4 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1482🥇1428🥇1534🥇1472🥇1425🥇1635
🥈2MI13771234🥈1454🥈1439🥈1365🥈1446
🥉3CART130712121343🥉1407🥉13071306
4ElasticNet1291🥉12521343137712361287
5MarkovBlanket128912181366131012711314
6Lasso1262🥈12751294131412141246
7(R)ReliefF12531120127113921184🥉1345
8F-Test123811601334129512451187
9GainRatio12291186🥉1369121611631243
10mRMR120311311278124911801205
11SBE115410901204112911841182
12LOCO110610811161107111321092
13SFS109210941117105411321066
14Random100010001000100010001000
15LaplacianScore93680910389468761000
diff --git a/data/subsets/tasks_multiclass/models_all/budgets_4/pareto.html b/data/subsets/tasks_multiclass/models_all/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..44e652a7e16c926d19b446508409f5515d700433 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, all models, budget 5
Hover or tap a method for its values.
10-310-210-11001011021031041050246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIF-TestElasticNetMarkovBlanket(R)ReliefFGainRatiomRMRLassoCARTSBESFSLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_all/budgets_4/per_model.csv b/data/subsets/tasks_multiclass/models_all/budgets_4/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..bf4b344d6970c1896da5c808112f98e06e354ebf --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_4/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1308.0,1357.0,1,1338.0,2,1252.0,2,1224.0,1,1429.0,1 +2,MI,Non-search,Information theory,1248.0,1191.0,4,1360.0,1,1247.0,3,1192.0,4,1284.0,3 +3,CART,Non-search,Tree,1235.0,1190.0,5,1291.0,5,1230.0,6,1214.0,2,1289.0,2 +4,F-Test,Non-search,Correlation,1216.0,1219.0,3,1266.0,7,1238.0,4,1162.0,5,1226.0,5 +5,ElasticNet,Non-search,Regularization,1200.0,1162.0,7,1266.0,8,1230.0,5,1201.0,3,1174.0,8 +6,Lasso,Non-search,Regularization,1180.0,1227.0,2,1266.0,6,1124.0,10,1141.0,6,1170.0,9 +7,MarkovBlanket,Search,Backward search,1173.0,1174.0,6,1296.0,4,1195.0,7,1021.0,11,1208.0,6 +8,(R)ReliefF,Non-search,Distance,1168.0,1162.0,8,1225.0,10,1260.0,1,1000.0,13,1226.0,4 +9,GainRatio,Non-search,Information theory,1144.0,1096.0,12,1315.0,3,1124.0,9,1026.0,10,1183.0,7 +10,mRMR,Non-search,Information theory,1120.0,1117.0,10,1210.0,11,1090.0,11,1083.0,7,1119.0,10 +11,SBE,Search,Backward search,1112.0,1117.0,11,1250.0,9,1068.0,12,1046.0,9,1097.0,11 +12,LaplacianScore,Non-search,Distance,1101.0,1088.0,14,1196.0,12,1149.0,8,1059.0,8,1029.0,13 +13,LOCO,Search,Leave-one-out,1062.0,1092.0,13,1145.0,13,1010.0,13,1017.0,12,1062.0,12 +14,SFS,Search,Forward search,1034.0,1121.0,9,1109.0,14,933.0,15,996.0,15,1010.0,14 +15,Random,Baseline,Baseline,1000.0,1000.0,15,1000.0,15,1000.0,14,1000.0,14,1000.0,15 diff --git a/data/subsets/tasks_multiclass/models_all/budgets_4/per_model_table.html b/data/subsets/tasks_multiclass/models_all/budgets_4/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..bb73808430f264fa7151733711aa38fe58374be6 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_4/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, all models, budget 5 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1308🥇1357🥈1338🥈1252🥇1224🥇1429
🥈2MI12481191🥇1360🥉12471192🥉1284
🥉3CART1235119012911230🥈1214🥈1289
4F-Test1216🥉12191266123811621226
5ElasticNet1200116212661230🥉12011174
6Lasso1180🥈12271266112411411170
7MarkovBlanket117311741296119510211208
8(R)ReliefF116811621225🥇126010001226
9GainRatio11441096🥉1315112410261183
10mRMR112011171210109010831119
11SBE111211171250106810461097
12LaplacianScore110110881196114910591029
13LOCO106210921145101010171062
14SFS1034112111099339961010
15Random100010001000100010001000
diff --git a/data/subsets/tasks_multiclass/models_all/budgets_all/pareto.html b/data/subsets/tasks_multiclass/models_all/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..34cedd579cd96e9009c2e99a3ddb41a8d614cb34 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, all models, all budgets
Hover or tap a method for its values.
10-210-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketCARTmRMRGainRatio(R)ReliefFElasticNetF-TestLassoSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_all/budgets_all/per_model.csv b/data/subsets/tasks_multiclass/models_all/budgets_all/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..1a1dc83f4c95642ebf35be6413d11dbc81d85bdc --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_all/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1431.0,1487.0,1,1448.0,1,1373.0,1,1382.0,1,1489.0,1 +2,CART,Non-search,Tree,1354.0,1349.0,2,1420.0,2,1301.0,3,1337.0,3,1379.0,3 +3,MI,Non-search,Information theory,1353.0,1325.0,4,1408.0,3,1317.0,2,1345.0,2,1388.0,2 +4,MarkovBlanket,Search,Backward search,1316.0,1337.0,3,1360.0,4,1278.0,4,1280.0,4,1337.0,4 +5,ElasticNet,Non-search,Regularization,1290.0,1306.0,6,1342.0,5,1245.0,6,1276.0,5,1296.0,6 +6,F-Test,Non-search,Correlation,1266.0,1266.0,7,1306.0,7,1233.0,7,1272.0,6,1266.0,7 +7,Lasso,Non-search,Regularization,1259.0,1316.0,5,1284.0,8,1224.0,8,1221.0,7,1265.0,8 +8,(R)ReliefF,Non-search,Distance,1252.0,1251.0,8,1270.0,10,1257.0,5,1196.0,9,1298.0,5 +9,mRMR,Non-search,Information theory,1230.0,1237.0,9,1284.0,9,1183.0,10,1197.0,8,1264.0,9 +10,GainRatio,Non-search,Information theory,1226.0,1218.0,10,1310.0,6,1205.0,9,1180.0,10,1230.0,10 +11,SBE,Search,Backward search,1147.0,1161.0,13,1187.0,11,1106.0,11,1139.0,11,1153.0,11 +12,LOCO,Search,Leave-one-out,1124.0,1169.0,12,1153.0,13,1062.0,12,1114.0,12,1129.0,12 +13,SFS,Search,Forward search,1120.0,1194.0,11,1154.0,12,1060.0,14,1108.0,13,1092.0,13 +14,LaplacianScore,Non-search,Distance,1004.0,968.0,15,1026.0,14,1062.0,13,972.0,15,979.0,15 +15,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,15,1000.0,15,1000.0,14,1000.0,14 diff --git a/data/subsets/tasks_multiclass/models_all/budgets_all/per_model_table.html b/data/subsets/tasks_multiclass/models_all/budgets_all/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..237406e4bf590d2b7d7ca382260852a96b2e9a18 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_all/budgets_all/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, all models, all budgets · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1431🥇1487🥇1448🥇1373🥇1382🥇1489
🥈2CART1354🥈1349🥈1420🥉1301🥉1337🥉1379
🥉3MI13531325🥉1408🥈1317🥈1345🥈1388
4MarkovBlanket1316🥉13371360127812801337
5ElasticNet129013061342124512761296
6F-Test126612661306123312721266
7Lasso125913161284122412211265
8(R)ReliefF125212511270125711961298
9mRMR123012371284118311971264
10GainRatio122612181310120511801230
11SBE114711611187110611391153
12LOCO112411691153106211141129
13SFS112011941154106011081092
14LaplacianScore100496810261062972979
15Random100010001000100010001000
diff --git a/data/subsets/tasks_multiclass/models_lightgbm/budgets_0/pareto.html b/data/subsets/tasks_multiclass/models_lightgbm/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..f9549e07768b204a5ea38dc06f0d9d64a35494ea --- /dev/null +++ b/data/subsets/tasks_multiclass/models_lightgbm/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LightGBM, budget 1
Hover or tap a method for its values.
10-210-110010110210310410505101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMarkovBlanketCARTElasticNetmRMR(R)ReliefFMILassoGainRatioF-TestSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_lightgbm/budgets_1/pareto.html b/data/subsets/tasks_multiclass/models_lightgbm/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..95438972ceb624f60630c5d300f61a5af8ff58e9 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_lightgbm/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LightGBM, budget 2
Hover or tap a method for its values.
10-310-210-110010110210310410505101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMarkovBlanketMICARTElasticNetGainRatioLasso(R)ReliefFF-TestmRMRLOCOSFSSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_lightgbm/budgets_2/pareto.html b/data/subsets/tasks_multiclass/models_lightgbm/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..b8f5cef922278462de06ab0a42a0158cc8c7ba6c --- /dev/null +++ b/data/subsets/tasks_multiclass/models_lightgbm/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LightGBM, budget 3
Hover or tap a method for its values.
10-310-210-11001011021031041050510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketGainRatioF-TestmRMR(R)ReliefFElasticNetLassoCARTSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_lightgbm/budgets_3/pareto.html b/data/subsets/tasks_multiclass/models_lightgbm/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..d474d0bfd21936c495bfb0203c02e6184aace8cc --- /dev/null +++ b/data/subsets/tasks_multiclass/models_lightgbm/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LightGBM, budget 4
Hover or tap a method for its values.
10-310-210-110010110210310410505101520Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketGainRatioF-TestLassomRMRElasticNetCART(R)ReliefFSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_lightgbm/budgets_4/pareto.html b/data/subsets/tasks_multiclass/models_lightgbm/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..96abcc2e24ac5f947c0ae57c65ac3e5c2174b47c --- /dev/null +++ b/data/subsets/tasks_multiclass/models_lightgbm/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LightGBM, budget 5
Hover or tap a method for its values.
10-310-210-11001011021031041050246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportance(R)ReliefFF-TestMarkovBlanketMIElasticNetLassomRMRGainRatioSBELOCOCARTRandomSFSDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_lightgbm/budgets_all/pareto.html b/data/subsets/tasks_multiclass/models_lightgbm/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..fe71c4f7cfc215bd9cd756818a325a2ddac3e9d1 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_lightgbm/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LightGBM, all budgets
Hover or tap a method for its values.
10-310-210-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMarkovBlanketMICARTmRMRElasticNet(R)ReliefFGainRatioLassoF-TestSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_linearmodel/budgets_0/pareto.html b/data/subsets/tasks_multiclass/models_linearmodel/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..817ad0c34b3a80382e6b993ed7141350da1ecb9d --- /dev/null +++ b/data/subsets/tasks_multiclass/models_linearmodel/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LinearModel, budget 1
Hover or tap a method for its values.
10-310-210-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalCARTRFImportanceMarkovBlanketMImRMRElasticNet(R)ReliefFLassoGainRatioF-TestSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_linearmodel/budgets_1/pareto.html b/data/subsets/tasks_multiclass/models_linearmodel/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..7c1d9427cbb96f39542b997905a52bea564f3b79 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_linearmodel/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LinearModel, budget 2
Hover or tap a method for its values.
10-310-210-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketCARTElasticNetGainRatio(R)ReliefFmRMRF-TestLassoLOCOSFSSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_linearmodel/budgets_2/pareto.html b/data/subsets/tasks_multiclass/models_linearmodel/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..cc1373793a53a7eae12d99463c0ae1ce1deb2294 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_linearmodel/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LinearModel, budget 3
Hover or tap a method for its values.
10-310-210-11001011021031041050510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIGainRatiomRMRElasticNetF-TestMarkovBlanket(R)ReliefFCARTLassoSBESFSLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_linearmodel/budgets_3/pareto.html b/data/subsets/tasks_multiclass/models_linearmodel/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..3fea01f304d508bc77d1a3d4907c574f5bd9b9d3 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_linearmodel/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LinearModel, budget 4
Hover or tap a method for its values.
10-310-210-11001011021031041050510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIGainRatioMarkovBlanketF-TestElasticNetmRMR(R)ReliefFCARTLassoSBELOCOSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_linearmodel/budgets_4/pareto.html b/data/subsets/tasks_multiclass/models_linearmodel/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..af888a56ffd15539a5a1d666d6f3e82f3bd22b12 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_linearmodel/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LinearModel, budget 5
Hover or tap a method for its values.
10-310-210-110010110210310410502468101214Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalMIGainRatioMarkovBlanketRFImportanceF-TestElasticNet(R)ReliefFmRMRLassoSBECARTLOCOSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_linearmodel/budgets_all/pareto.html b/data/subsets/tasks_multiclass/models_linearmodel/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..6eca413e7d4809e2161b16308b97b6168a10c2f7 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_linearmodel/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, LinearModel, all budgets
Hover or tap a method for its values.
10-310-210-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketCARTmRMRElasticNetGainRatio(R)ReliefFF-TestLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_randomforest/budgets_0/pareto.html b/data/subsets/tasks_multiclass/models_randomforest/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..da8738cbf4348914b8732f913d30ae83a8d8b4c1 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_randomforest/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RandomForest, budget 1
Hover or tap a method for its values.
10-310-210-110010110210310410505101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalGainRatioMarkovBlanketMIRFImportanceCART(R)ReliefFF-TestLassoSFSElasticNetmRMRSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_randomforest/budgets_1/pareto.html b/data/subsets/tasks_multiclass/models_randomforest/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..2396243954c623e142f12f379d2e50df0c652e70 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_randomforest/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RandomForest, budget 2
Hover or tap a method for its values.
10-310-210-110010110210310410505101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketCARTGainRatioLassoF-Test(R)ReliefFElasticNetmRMRSBESFSLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_randomforest/budgets_2/pareto.html b/data/subsets/tasks_multiclass/models_randomforest/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..bd9144631de750ee99be56176f8d2bd18b0e19bd --- /dev/null +++ b/data/subsets/tasks_multiclass/models_randomforest/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RandomForest, budget 3
Hover or tap a method for its values.
10-310-210-1100101102103104105-5051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMImRMRMarkovBlanket(R)ReliefFElasticNetCARTGainRatioF-TestLassoSFSSBERandomLOCODominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_randomforest/budgets_3/pareto.html b/data/subsets/tasks_multiclass/models_randomforest/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..96ac4d435a7eef6e909d655d98376e2c0a63a4e0 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_randomforest/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RandomForest, budget 4
Hover or tap a method for its values.
10-310-210-11001011021031041050510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalMIRFImportance(R)ReliefFLassoElasticNetmRMRMarkovBlanketCARTF-TestGainRatioSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_randomforest/budgets_4/pareto.html b/data/subsets/tasks_multiclass/models_randomforest/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..7f9b6aae6ed0c11c5f6930e19aba30a0d338b922 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_randomforest/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RandomForest, budget 5
Hover or tap a method for its values.
10-310-210-1100101102103104105024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalF-TestRFImportance(R)ReliefFElasticNetMICARTLassomRMRGainRatioMarkovBlanketSBERandomSFSLOCODominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_randomforest/budgets_all/pareto.html b/data/subsets/tasks_multiclass/models_randomforest/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..275296539475a93b65078aa3c48595878379300f --- /dev/null +++ b/data/subsets/tasks_multiclass/models_randomforest/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RandomForest, all budgets
Hover or tap a method for its values.
10-310-210-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketCARTGainRatio(R)ReliefFF-TestmRMRElasticNetLassoSFSSBELOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_0/elo_explorer.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_0/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..ed5c7958c35f0a0ffc1f2d789f48541a84705921 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_0/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 1
Hover or tap a method for its values.
80010001200140016001800EloLaplacianScoreRandomLOCOSBESFS(R)ReliefFLassoGainRatiomRMRElasticNetF-TestRFImportanceMICARTMarkovBlanketstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_0/leaderboard_table.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_0/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..7552962b2041ec8f6f817661f76080b727a4fe80 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_0/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 1
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1MarkovBlanketBackward searchsearch1509+157 / −1005.0618.70.4730.4910.20.0
🥈2CARTTreenon-search1504+214 / −1364.8619.70.5360.8270.00.0
🥉3MIInformation theorynon-search1443+123 / −975.9424.80.7340.9880.20.0
4RFImportanceTreenon-search1427+100 / −735.9620.30.6800.9600.10.0
5F-TestCorrelationnon-search1372+118 / −928.0130.80.7581.0000.00.0
6ElasticNetRegularizationnon-search1353+175 / −1506.9927.20.5850.97519.10.0
7mRMRInformation theorynon-search1332+107 / −937.4425.00.6150.9580.40.0
8GainRatioInformation theorynon-search1320+147 / −1548.2829.70.7180.9720.10.0
9LassoRegularizationnon-search1284+146 / −1377.7230.30.4780.9707.40.0
10(R)ReliefFDistancenon-search1283+147 / −1528.3529.20.6820.9882.70.0
11SFSForward searchsearch1251+116 / −1408.6929.20.4100.7092,036.019.4
12SBEBackward searchsearch1232+135 / −1679.7933.70.1950.5803,601.261.1
13LOCOLeave-one-outsearch1214+94 / −998.6032.50.2940.719347.30.0
14RandomBaseline1000+90 / −17211.7243.4-0.0060.4300.00.0
15LaplacianScoreDistancenon-search960+121 / −25512.5847.60.4990.8490.20.0
diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_0/pareto.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_0/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..b4659edc34e631c54ec17b77c30c5b1d14d1f6fc --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_0/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,33.094,28.7378,12,True +MarkovBlanket,31.6236,12.6532,12,True +MI,28.7563,23.3191,12,False +F-Test,25.6216,1.6201,12,True +ElasticNet,25.0905,4208.9807,12,False +CART,24.7618,10.6247,12,False +mRMR,24.7375,8.7991,12,False +(R)ReliefF,24.6204,30687.3259,12,False +GainRatio,24.2437,3.9312,12,False +SFS,20.3126,28635.3556,12,False +Lasso,19.0419,6282.4714,12,False +LOCO,16.1023,9907.0686,12,False +SBE,15.1491,50403.7434,12,False +Random,0.0,0.0312,12,True +LaplacianScore,-8.0975,4.2563,12,False diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_0/pareto.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..3606ccacd6b239d8462edf9e4e98f3c02224aca4 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 1
Hover or tap a method for its values.
10-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMarkovBlanketMIF-TestElasticNetCARTmRMR(R)ReliefFGainRatioSFSLassoLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_0/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_0/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..6f15853b39e33964498eb61ac13d81ce64b3640f --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_0/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,MarkovBlanket,Search,Backward search,1509.0,1409.0,1666.0,5.06,71.0,18.7,,0.473,0.491,0.2,0.0,12,36 +2,CART,Non-search,Tree,1504.0,1368.0,1718.0,4.86,72.4,19.7,,0.536,0.827,0.0,0.0,12,36 +3,MI,Non-search,Information theory,1443.0,1346.0,1566.0,5.94,64.7,24.8,,0.734,0.988,0.2,0.0,12,36 +4,RFImportance,Non-search,Tree,1427.0,1354.0,1527.0,5.96,64.6,20.3,,0.68,0.96,0.1,0.0,12,36 +5,F-Test,Non-search,Correlation,1372.0,1280.0,1490.0,8.01,49.9,30.8,,0.758,1.0,0.0,0.0,12,36 +6,ElasticNet,Non-search,Regularization,1353.0,1203.0,1528.0,6.99,57.2,27.2,,0.585,0.975,19.1,0.0,12,36 +7,mRMR,Non-search,Information theory,1332.0,1239.0,1439.0,7.44,54.0,25.0,,0.615,0.958,0.4,0.0,12,36 +8,GainRatio,Non-search,Information theory,1320.0,1166.0,1467.0,8.28,48.0,29.7,,0.718,0.972,0.1,0.0,12,36 +9,Lasso,Non-search,Regularization,1284.0,1147.0,1430.0,7.72,52.0,30.3,,0.478,0.97,7.4,0.0,12,36 +10,(R)ReliefF,Non-search,Distance,1283.0,1131.0,1430.0,8.35,47.5,29.2,,0.682,0.988,2.7,0.0,12,36 +11,SFS,Search,Forward search,1251.0,1111.0,1367.0,8.69,45.0,29.2,,0.41,0.709,2036.0,19.4,12,36 +12,SBE,Search,Backward search,1232.0,1065.0,1367.0,9.79,37.2,33.7,,0.195,0.58,3601.2,61.1,12,36 +13,LOCO,Search,Leave-one-out,1214.0,1115.0,1308.0,8.6,45.7,32.5,,0.294,0.719,347.3,0.0,12,36 +14,Random,Baseline,Baseline,1000.0,828.0,1090.0,11.72,23.4,43.4,,-0.006,0.43,0.0,0.0,12,36 +15,LaplacianScore,Non-search,Distance,960.0,705.0,1081.0,12.58,17.3,47.6,,0.499,0.849,0.2,0.0,12,36 diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_1/elo_explorer.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_1/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..1d1bbcbc0d7c80464e83ad4ce220adfc53b392bc --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_1/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 2
Hover or tap a method for its values.
80010001200140016001800EloRandomLaplacianScoreSBESFSLOCOGainRatiomRMRF-TestLassoElasticNet(R)ReliefFMarkovBlanketMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_1/leaderboard_table.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_1/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..b6e98219723fa0f8182ffdaaef764404481cab95 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_1/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 2
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1516+113 / −675.1517.70.6910.8740.10.0
🥈2CARTTreenon-search1511+194 / −1354.8117.90.4600.6930.00.0
🥉3MIInformation theorynon-search1450+150 / −1165.7822.50.7250.9780.20.0
4MarkovBlanketBackward searchsearch1445+107 / −736.0721.40.4630.4960.20.0
5(R)ReliefFDistancenon-search1412+165 / −1307.1122.70.6950.9662.80.0
6ElasticNetRegularizationnon-search1402+171 / −1256.7125.00.6020.95919.10.0
7LassoRegularizationnon-search1357+173 / −1587.4326.30.5080.9157.40.0
8F-TestCorrelationnon-search1352+124 / −1107.5029.40.7850.9860.00.0
9mRMRInformation theorynon-search1330+137 / −1088.0728.30.6780.8740.50.0
10GainRatioInformation theorynon-search1312+134 / −1378.0427.50.6970.9760.10.0
11LOCOLeave-one-outsearch1218+124 / −1349.1130.90.2060.654352.50.0
12SFSForward searchsearch1213+90 / −1179.8331.20.3240.6543,600.141.7
13SBEBackward searchsearch1173+124 / −18210.3832.80.1390.5923,601.358.3
14LaplacianScoreDistancenon-search1037+160 / −35911.7845.30.5570.8430.20.0
15RandomBaseline1000+89 / −15412.2440.70.0020.4850.00.0
diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_1/pareto.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_1/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..125e48e86d88166c816a5a19c24e5608a6c6657c --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_1/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,32.9779,28.6621,12,True +MI,29.7417,23.5018,12,True +MarkovBlanket,29.7156,12.8377,12,True +CART,29.3061,10.5047,12,True +GainRatio,24.3466,3.4586,12,True +(R)ReliefF,23.7512,30685.7114,12,False +ElasticNet,22.6997,4429.8433,12,False +F-Test,21.8048,0.5869,12,True +mRMR,20.3182,15.0713,12,False +LOCO,18.6285,9875.357,12,False +SFS,16.418,42300.084,12,False +Lasso,14.8415,6291.2042,12,False +SBE,11.3201,48584.4594,12,False +Random,0.0,0.0029,12,True +LaplacianScore,-35.4215,4.137,12,False diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_1/pareto.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..fbdfc67917abf7084b38f6abb99c9c5a1b041f10 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 2
Hover or tap a method for its values.
10-210-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketCARTGainRatio(R)ReliefFElasticNetF-TestmRMRLOCOSFSLassoSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_1/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_1/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..00dce05e90544c11fc7cef690c17a3b25996a4ba --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_1/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1516.0,1449.0,1629.0,5.15,70.3,17.7,,0.691,0.874,0.1,0.0,12,36 +2,CART,Non-search,Tree,1511.0,1376.0,1705.0,4.81,72.8,17.9,,0.46,0.693,0.0,0.0,12,36 +3,MI,Non-search,Information theory,1450.0,1334.0,1600.0,5.78,65.9,22.5,,0.725,0.978,0.2,0.0,12,36 +4,MarkovBlanket,Search,Backward search,1445.0,1372.0,1552.0,6.07,63.8,21.4,,0.463,0.496,0.2,0.0,12,36 +5,(R)ReliefF,Non-search,Distance,1412.0,1282.0,1577.0,7.11,56.4,22.7,,0.695,0.966,2.8,0.0,12,36 +6,ElasticNet,Non-search,Regularization,1402.0,1277.0,1573.0,6.71,59.2,25.0,,0.602,0.959,19.1,0.0,12,36 +7,Lasso,Non-search,Regularization,1357.0,1199.0,1530.0,7.43,54.1,26.3,,0.508,0.915,7.4,0.0,12,36 +8,F-Test,Non-search,Correlation,1352.0,1242.0,1476.0,7.5,53.6,29.4,,0.785,0.986,0.0,0.0,12,36 +9,mRMR,Non-search,Information theory,1330.0,1222.0,1467.0,8.07,49.5,28.3,,0.678,0.874,0.5,0.0,12,36 +10,GainRatio,Non-search,Information theory,1312.0,1175.0,1446.0,8.04,49.7,27.5,,0.697,0.976,0.1,0.0,12,36 +11,LOCO,Search,Leave-one-out,1218.0,1084.0,1342.0,9.11,42.1,30.9,,0.206,0.654,352.5,0.0,12,36 +12,SFS,Search,Forward search,1213.0,1096.0,1303.0,9.83,36.9,31.2,,0.324,0.654,3600.1,41.7,12,36 +13,SBE,Search,Backward search,1173.0,991.0,1297.0,10.38,33.0,32.8,,0.139,0.592,3601.3,58.3,12,36 +14,LaplacianScore,Non-search,Distance,1037.0,678.0,1197.0,11.78,23.0,45.3,,0.557,0.843,0.2,0.0,12,36 +15,Random,Baseline,Baseline,1000.0,846.0,1089.0,12.24,19.7,40.7,,0.002,0.485,0.0,0.0,12,36 diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_2/elo_explorer.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_2/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..7209c000647aac64950a55ad16eaaf96b66da415 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_2/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 3
Hover or tap a method for its values.
60080010001200140016001800EloLaplacianScoreRandomSFSLOCOSBEmRMRGainRatioLasso(R)ReliefFCARTMarkovBlanketElasticNetF-TestMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_2/leaderboard_table.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_2/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..fb928495f2395265398fe43e0282d4193b651d43 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_2/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 3
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1436+166 / −934.8615.60.7050.8080.10.0
🥈2MIInformation theorynon-search1383+155 / −1075.6919.50.7220.9120.20.0
🥉3F-TestCorrelationnon-search1320+144 / −1056.0320.40.7490.9390.00.0
4ElasticNetRegularizationnon-search1282+172 / −1356.5024.30.5720.88619.10.0
5MarkovBlanketBackward searchsearch1272+121 / −846.7823.70.4780.4870.20.0
6CARTTreenon-search1254+130 / −1326.4723.10.4300.6380.00.0
7LassoRegularizationnon-search1190+176 / −1667.4428.60.5180.8057.50.0
8(R)ReliefFDistancenon-search1190+145 / −1367.6423.60.7060.8932.80.0
9GainRatioInformation theorynon-search1163+93 / −968.3125.50.7320.8960.10.0
10mRMRInformation theorynon-search1141+103 / −958.1926.00.6650.8150.80.0
11SBEBackward searchsearch1122+136 / −1359.2226.00.1410.5073,601.455.6
12LOCOLeave-one-outsearch1056+120 / −1389.5629.80.1430.618327.411.1
13SFSForward searchsearch1011+58 / −1039.6128.20.3260.6373,600.252.8
14RandomBaseline1000+131 / −19911.2536.9-0.0010.5140.00.0
15LaplacianScoreDistancenon-search876+136 / −29612.4443.90.6020.8240.20.0
diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_2/pareto.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_2/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..3364eb91b8151df176295516563a3bf5f4cb1516 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_2/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,26.2861,28.6782,12,True +MI,26.0528,23.7364,12,True +GainRatio,21.2721,3.4985,12,True +MarkovBlanket,21.072,12.497,12,False +F-Test,19.7156,0.5739,12,True +CART,17.7877,10.4921,12,False +mRMR,17.3479,27.9866,12,False +(R)ReliefF,15.8724,30689.0831,12,False +SFS,11.4066,52692.5412,12,False +ElasticNet,10.8795,4321.0708,12,False +SBE,10.1253,44015.253,12,False +LOCO,3.8584,9710.4168,12,False +Random,0.0,0.0033,12,True +Lasso,-1.1784,6309.2164,12,False +LaplacianScore,-41.2672,6.1586,12,False diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_2/pareto.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..7078dd7857f03b4e95677cd605ae41f2b83be7ef --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 3
Hover or tap a method for its values.
10-210-11001011021031041050510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIGainRatioMarkovBlanketF-TestCARTmRMR(R)ReliefFSFSElasticNetSBELOCORandomLassoDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_2/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_2/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..2a45894892b5ae5f5a770e76c01b77e4b7a0db71 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_2/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1436.0,1343.0,1602.0,4.86,72.4,15.6,,0.705,0.808,0.1,0.0,12,36 +2,MI,Non-search,Information theory,1383.0,1276.0,1538.0,5.69,66.5,19.5,,0.722,0.912,0.2,0.0,12,36 +3,F-Test,Non-search,Correlation,1320.0,1215.0,1464.0,6.03,64.1,20.4,,0.749,0.939,0.0,0.0,12,36 +4,ElasticNet,Non-search,Regularization,1282.0,1147.0,1454.0,6.5,60.7,24.3,,0.572,0.886,19.1,0.0,12,36 +5,MarkovBlanket,Search,Backward search,1272.0,1188.0,1393.0,6.78,58.7,23.7,,0.478,0.487,0.2,0.0,12,36 +6,CART,Non-search,Tree,1254.0,1122.0,1384.0,6.47,60.9,23.1,,0.43,0.638,0.0,0.0,12,36 +7,Lasso,Non-search,Regularization,1190.0,1024.0,1366.0,7.44,54.0,28.6,,0.518,0.805,7.5,0.0,12,36 +8,(R)ReliefF,Non-search,Distance,1190.0,1054.0,1335.0,7.64,52.6,23.6,,0.706,0.893,2.8,0.0,12,36 +9,GainRatio,Non-search,Information theory,1163.0,1067.0,1256.0,8.31,47.8,25.5,,0.732,0.896,0.1,0.0,12,36 +10,mRMR,Non-search,Information theory,1141.0,1046.0,1244.0,8.19,48.6,26.0,,0.665,0.815,0.8,0.0,12,36 +11,SBE,Search,Backward search,1122.0,987.0,1258.0,9.22,41.3,26.0,,0.141,0.507,3601.4,55.6,12,36 +12,LOCO,Search,Leave-one-out,1056.0,918.0,1176.0,9.56,38.9,29.8,,0.143,0.618,327.4,11.1,12,36 +13,SFS,Search,Forward search,1011.0,908.0,1069.0,9.61,38.5,28.2,,0.326,0.637,3600.2,52.8,12,36 +14,Random,Baseline,Baseline,1000.0,801.0,1131.0,11.25,26.8,36.9,,-0.001,0.514,0.0,0.0,12,36 +15,LaplacianScore,Non-search,Distance,876.0,580.0,1012.0,12.44,18.2,43.9,,0.602,0.824,0.2,0.0,12,36 diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_3/elo_explorer.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_3/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..d8e55cd4ec7fa9777682222746ba1ecae45b5118 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_3/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 4
Hover or tap a method for its values.
6008001000120014001600EloLaplacianScoreRandomLOCOSFSGainRatiomRMRSBE(R)ReliefFLassoElasticNetF-TestMarkovBlanketCARTMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_3/leaderboard_table.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_3/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..91a71dbfd72e2b8df42be655f3790c037a393cb2 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_3/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 4
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1425+167 / −1174.4414.20.7050.6850.10.0
🥈2MIInformation theorynon-search1365+131 / −925.5015.30.7040.7970.20.0
🥉3CARTTreenon-search1307+128 / −1176.5020.70.3800.5790.00.0
4MarkovBlanketBackward searchsearch1271+133 / −1066.9718.10.5020.4790.20.0
5F-TestCorrelationnon-search1245+111 / −977.4222.60.6760.8140.00.0
6ElasticNetRegularizationnon-search1236+130 / −1206.6421.20.5470.78219.10.0
7LassoRegularizationnon-search1214+124 / −1127.3324.50.4550.7097.50.0
8SBEBackward searchsearch1184+143 / −1488.1423.60.1060.5103,149.644.4
9(R)ReliefFDistancenon-search1184+102 / −938.3925.00.6640.7622.80.0
10mRMRInformation theorynon-search1180+134 / −1228.5623.60.6750.6920.98.3
11GainRatioInformation theorynon-search1163+122 / −1098.8322.40.7330.7530.10.0
12LOCOLeave-one-outsearch1132+136 / −1628.3624.70.1490.575299.616.7
13SFSForward searchsearch1132+123 / −1489.6125.20.2670.5663,600.561.1
14RandomBaseline1000+131 / −18110.9232.30.0020.4860.00.0
15LaplacianScoreDistancenon-search876+136 / −26812.3936.80.6630.7330.20.0
diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_3/pareto.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_3/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..8654c69b37e0cb7e8686b59f4d2b8ecf3c77fb54 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_3/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +MI,23.872,25.6349,12,True +CART,21.9621,11.12,12,True +RFImportance,21.4354,29.8448,12,False +MarkovBlanket,19.0524,12.1544,12,False +mRMR,18.4386,56.1673,12,False +F-Test,15.9595,0.5781,12,True +ElasticNet,15.45,4369.8415,12,False +LOCO,15.3951,9304.6932,12,False +SFS,14.5873,52633.0277,12,False +GainRatio,14.4701,3.4302,12,False +SBE,13.2969,35416.8452,12,False +(R)ReliefF,12.8533,30688.5686,12,False +Lasso,12.5846,6230.3566,12,False +Random,0.0,0.0022,12,True +LaplacianScore,-14.4357,4.1886,12,False diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_3/pareto.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..f4c8844df0616dd95ae432a74f1a2b6b9c630818 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 4
Hover or tap a method for its values.
10-310-210-11001011021031041050510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalMICARTRFImportanceMarkovBlanketmRMRF-TestElasticNetLOCOSFSGainRatioSBE(R)ReliefFLassoRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_3/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_3/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..3da68d52708cc3c02bd2e44b3da1255372f0f236 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_3/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1425.0,1308.0,1592.0,4.44,75.4,14.2,,0.705,0.685,0.1,0.0,12,36 +2,MI,Non-search,Information theory,1365.0,1273.0,1496.0,5.5,67.9,15.3,,0.704,0.797,0.2,0.0,12,36 +3,CART,Non-search,Tree,1307.0,1190.0,1435.0,6.5,60.7,20.7,,0.38,0.579,0.0,0.0,12,36 +4,MarkovBlanket,Search,Backward search,1271.0,1165.0,1404.0,6.97,57.3,18.1,,0.502,0.479,0.2,0.0,12,36 +5,F-Test,Non-search,Correlation,1245.0,1148.0,1356.0,7.42,54.2,22.6,,0.676,0.814,0.0,0.0,12,36 +6,ElasticNet,Non-search,Regularization,1236.0,1116.0,1366.0,6.64,59.7,21.2,,0.547,0.782,19.1,0.0,12,36 +7,Lasso,Non-search,Regularization,1214.0,1102.0,1338.0,7.33,54.8,24.5,,0.455,0.709,7.5,0.0,12,36 +8,SBE,Search,Backward search,1184.0,1036.0,1327.0,8.14,49.0,23.6,,0.106,0.51,3149.6,44.4,12,36 +9,(R)ReliefF,Non-search,Distance,1184.0,1091.0,1286.0,8.39,47.2,25.0,,0.664,0.762,2.8,0.0,12,36 +10,mRMR,Non-search,Information theory,1180.0,1058.0,1314.0,8.56,46.0,23.6,,0.675,0.692,0.9,8.3,12,36 +11,GainRatio,Non-search,Information theory,1163.0,1054.0,1285.0,8.83,44.0,22.4,,0.733,0.753,0.1,0.0,12,36 +12,LOCO,Search,Leave-one-out,1132.0,970.0,1268.0,8.36,47.4,24.7,,0.149,0.575,299.6,16.7,12,36 +13,SFS,Search,Forward search,1132.0,984.0,1255.0,9.61,38.5,25.2,,0.267,0.566,3600.5,61.1,12,36 +14,Random,Baseline,Baseline,1000.0,819.0,1131.0,10.92,29.2,32.3,,0.002,0.486,0.0,0.0,12,36 +15,LaplacianScore,Non-search,Distance,876.0,608.0,1012.0,12.39,18.6,36.8,,0.663,0.733,0.2,0.0,12,36 diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_4/elo_explorer.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_4/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..d2ec3e3061174bfc892008dff8b7ae8b2ddfe5cd --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_4/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 5
Hover or tap a method for its values.
80090010001100120013001400EloSFS(R)ReliefFRandomLOCOMarkovBlanketGainRatioSBELaplacianScoremRMRLassoF-TestMIElasticNetCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_4/leaderboard_table.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_4/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..b0f5e3dab69731e886ff11458ae14a0065d4d0ab --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_4/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 5
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1224+133 / −966.4716.20.6360.5530.10.0
🥈2CARTTreenon-search1214+151 / −1265.9716.40.3680.5180.00.0
🥉3ElasticNetRegularizationnon-search1201+140 / −1106.6715.80.4280.63919.00.0
4MIInformation theorynon-search1192+122 / −1046.9215.80.6310.6760.20.0
5F-TestCorrelationnon-search1162+135 / −1076.9216.50.6230.7070.00.0
6LassoRegularizationnon-search1141+108 / −1026.5318.80.3480.6107.40.0
7mRMRInformation theorynon-search1083+113 / −1138.0816.80.5760.6191.08.3
8LaplacianScoreDistancenon-search1059+157 / −1488.6918.80.6080.6630.20.0
9SBEBackward searchsearch1046+144 / −1508.7121.30.0750.5141,741.941.7
10GainRatioInformation theorynon-search1026+96 / −1189.1719.30.6010.6680.10.0
11MarkovBlanketBackward searchsearch1021+161 / −1728.9418.70.4000.4680.10.0
12LOCOLeave-one-outsearch1017+118 / −1318.8120.10.1130.518300.516.7
13(R)ReliefFDistancenon-search1000+87 / −998.8921.10.6480.6532.70.0
14RandomBaseline1000+115 / −13810.0421.70.0040.5040.00.0
15SFSForward searchsearch996+98 / −1209.1921.80.2910.5163,600.963.9
diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_4/pareto.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_4/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..171c8a36a3ed97c59b6d1fdc0238cef83bb40692 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_4/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +ElasticNet,9.1059,4435.9526,12,True +MI,8.9808,23.7896,12,True +RFImportance,8.0681,28.9855,12,False +MarkovBlanket,6.2224,11.8512,12,True +F-Test,6.1144,0.5907,12,True +mRMR,5.8558,95.8996,12,False +GainRatio,4.9755,3.4694,12,False +SBE,2.2628,22955.0306,12,False +CART,2.1016,10.5066,12,False +Lasso,2.0812,6217.2719,12,False +SFS,0.8086,55646.5336,12,False +(R)ReliefF,0.187,30688.0563,12,False +Random,0.0,0.0029,12,True +LaplacianScore,-1.0705,4.9177,12,False +LOCO,-2.6255,9307.2216,12,False diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_4/pareto.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..38fbd9cf6b41d2c7fff4e4beb921fbcbdfc9dcc6 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RealMLP, budget 5
Hover or tap a method for its values.
10-210-1100101102103104105-202468Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalElasticNetMIRFImportanceMarkovBlanketF-TestmRMRGainRatioSBECARTLassoSFS(R)ReliefFRandomLOCODominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_4/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_4/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..ab695a2b9baf73d3e165846bee4dcaee3e3de643 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_4/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1224.0,1128.0,1357.0,6.47,60.9,16.2,,0.636,0.553,0.1,0.0,12,36 +2,CART,Non-search,Tree,1214.0,1088.0,1365.0,5.97,64.5,16.4,,0.368,0.518,0.0,0.0,12,36 +3,ElasticNet,Non-search,Regularization,1201.0,1091.0,1341.0,6.67,59.5,15.8,,0.428,0.639,19.0,0.0,12,36 +4,MI,Non-search,Information theory,1192.0,1088.0,1314.0,6.92,57.7,15.8,,0.631,0.676,0.2,0.0,12,36 +5,F-Test,Non-search,Correlation,1162.0,1055.0,1297.0,6.92,57.7,16.5,,0.623,0.707,0.0,0.0,12,36 +6,Lasso,Non-search,Regularization,1141.0,1039.0,1249.0,6.53,60.5,18.8,,0.348,0.61,7.4,0.0,12,36 +7,mRMR,Non-search,Information theory,1083.0,970.0,1196.0,8.08,49.4,16.8,,0.576,0.619,1.0,8.3,12,36 +8,LaplacianScore,Non-search,Distance,1059.0,911.0,1216.0,8.69,45.0,18.8,,0.608,0.663,0.2,0.0,12,36 +9,SBE,Search,Backward search,1046.0,896.0,1190.0,8.71,44.9,21.3,,0.075,0.514,1741.9,41.7,12,36 +10,GainRatio,Non-search,Information theory,1026.0,908.0,1122.0,9.17,41.7,19.3,,0.601,0.668,0.1,0.0,12,36 +11,MarkovBlanket,Search,Backward search,1021.0,849.0,1182.0,8.94,43.2,18.7,,0.4,0.468,0.1,0.0,12,36 +12,LOCO,Search,Leave-one-out,1017.0,886.0,1135.0,8.81,44.2,20.1,,0.113,0.518,300.5,16.7,12,36 +13,(R)ReliefF,Non-search,Distance,1000.0,901.0,1087.0,8.89,43.6,21.1,,0.648,0.653,2.7,0.0,12,36 +14,Random,Baseline,Baseline,1000.0,862.0,1115.0,10.04,35.4,21.7,,0.004,0.504,0.0,0.0,12,36 +15,SFS,Search,Forward search,996.0,876.0,1094.0,9.19,41.5,21.8,,0.291,0.516,3600.9,63.9,12,36 diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_all/elo_explorer.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_all/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..b8d60349589883f71f39e887fed985416333ad65 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_all/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, RealMLP, all budgets
Hover or tap a method for its values.
800900100011001200130014001500EloLaplacianScoreRandomSFSLOCOSBEGainRatio(R)ReliefFmRMRLassoF-TestElasticNetMarkovBlanketCARTMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_all/leaderboard_table.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_all/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..9932d881060554d59defdbbc262439dabd98d929 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_all/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, RealMLP, all budgets
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1382+74 / −675.3816.80.6830.7760.10.0
🥈2MIInformation theorynon-search1345+88 / −755.9719.60.7030.8700.20.0
🥉3CARTTreenon-search1337+113 / −985.7219.60.4350.6510.00.0
4MarkovBlanketBackward searchsearch1280+71 / −576.7620.10.4630.4840.20.0
5ElasticNetRegularizationnon-search1276+100 / −926.7022.70.5470.84819.00.0
6F-TestCorrelationnon-search1272+76 / −717.1823.90.7180.8890.00.0
7LassoRegularizationnon-search1221+107 / −937.2925.70.4610.8027.40.0
8mRMRInformation theorynon-search1197+91 / −828.0723.90.6420.7920.63.3
9(R)ReliefFDistancenon-search1196+77 / −788.0724.30.6790.8522.70.0
10GainRatioInformation theorynon-search1180+79 / −918.5224.90.6960.8530.10.0
11SBEBackward searchsearch1139+90 / −1079.2527.50.1260.5413,600.552.2
12LOCOLeave-one-outsearch1114+91 / −968.8927.60.1810.617328.48.9
13SFSForward searchsearch1108+66 / −1009.3927.10.3310.6163,600.047.8
14RandomBaseline1000+84 / −10711.2335.00.0000.4840.00.0
15LaplacianScoreDistancenon-search972+93 / −13211.5838.50.5860.7820.20.0
diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_all/pareto.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_all/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..27e8392c20dc99412745938b4c9e8ad5047eb600 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_all/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,30.2519,28.9817,12,True +MI,28.0626,23.9964,12,True +MarkovBlanket,27.4198,12.3987,12,True +CART,24.6592,10.6496,12,True +mRMR,22.8896,40.7848,12,False +GainRatio,22.8636,3.5576,12,True +F-Test,22.5324,0.79,12,True +ElasticNet,22.1359,4353.1378,12,False +(R)ReliefF,21.7305,30687.749,12,False +SFS,17.629,46381.5084,12,False +LOCO,15.5849,9620.9514,12,False +Lasso,15.548,6266.1041,12,False +SBE,13.4939,40275.0663,12,False +Random,0.0,0.0085,12,True +LaplacianScore,-11.8049,4.7316,12,False diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_all/pareto.html b/data/subsets/tasks_multiclass/models_realmlp/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..f38758695acd131edc0ca8c391f92499b592d059 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, RealMLP, all budgets
Hover or tap a method for its values.
10-210-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketCARTmRMRGainRatioF-TestElasticNet(R)ReliefFSFSLOCOLassoSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_realmlp/budgets_all/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_realmlp/budgets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..5ed3b74b3825485b379e07867fcb387a00395733 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_realmlp/budgets_all/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1382.0,1315.0,1456.0,5.38,68.7,16.8,,0.683,0.776,0.1,0.0,12,180 +2,MI,Non-search,Information theory,1345.0,1270.0,1433.0,5.97,64.5,19.6,,0.703,0.87,0.2,0.0,12,180 +3,CART,Non-search,Tree,1337.0,1239.0,1450.0,5.72,66.3,19.6,,0.435,0.651,0.0,0.0,12,180 +4,MarkovBlanket,Search,Backward search,1280.0,1223.0,1351.0,6.76,58.8,20.1,,0.463,0.484,0.2,0.0,12,180 +5,ElasticNet,Non-search,Regularization,1276.0,1184.0,1376.0,6.7,59.3,22.7,,0.547,0.848,19.0,0.0,12,180 +6,F-Test,Non-search,Correlation,1272.0,1201.0,1348.0,7.18,55.9,23.9,,0.718,0.889,0.0,0.0,12,180 +7,Lasso,Non-search,Regularization,1221.0,1128.0,1328.0,7.29,55.1,25.7,,0.461,0.802,7.4,0.0,12,180 +8,mRMR,Non-search,Information theory,1197.0,1115.0,1288.0,8.07,49.5,23.9,,0.642,0.792,0.6,3.3,12,180 +9,(R)ReliefF,Non-search,Distance,1196.0,1118.0,1273.0,8.07,49.5,24.3,,0.679,0.852,2.7,0.0,12,180 +10,GainRatio,Non-search,Information theory,1180.0,1089.0,1259.0,8.52,46.2,24.9,,0.696,0.853,0.1,0.0,12,180 +11,SBE,Search,Backward search,1139.0,1032.0,1229.0,9.25,41.1,27.5,,0.126,0.541,3600.5,52.2,12,180 +12,LOCO,Search,Leave-one-out,1114.0,1018.0,1205.0,8.89,43.7,27.6,,0.181,0.617,328.4,8.9,12,180 +13,SFS,Search,Forward search,1108.0,1008.0,1174.0,9.39,40.1,27.1,,0.331,0.616,3600.0,47.8,12,180 +14,Random,Baseline,Baseline,1000.0,893.0,1084.0,11.23,26.9,35.0,,0.0,0.484,0.0,0.0,12,180 +15,LaplacianScore,Non-search,Distance,972.0,840.0,1065.0,11.58,24.4,38.5,,0.586,0.782,0.2,0.0,12,180 diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/elo_explorer.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..7eaaf688365d2d83fa87f6922f96bca60352ce10 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 1
Hover or tap a method for its values.
60080010001200140016001800EloLaplacianScoreRandomSFSLOCOSBE(R)ReliefFGainRatioLassoF-TestmRMRElasticNetMIRFImportanceMarkovBlanketCARTstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/leaderboard_table.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..9e368637404933150e1a9be4d273a5efac552a3a --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 1
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1CARTTreenon-search1543+152 / −894.9918.70.5360.8270.00.0
🥈2MarkovBlanketBackward searchsearch1520+153 / −1045.0315.60.4730.4910.20.0
🥉3RFImportanceTreenon-search1499+134 / −995.4614.30.6800.9600.10.0
4MIInformation theorynon-search1460+103 / −856.0724.90.7340.9880.20.0
5ElasticNetRegularizationnon-search1391+187 / −1346.8824.50.5850.97519.10.0
6mRMRInformation theorynon-search1387+122 / −1037.2421.80.6150.9580.30.0
7F-TestCorrelationnon-search1382+120 / −877.9232.30.7581.0000.00.0
8LassoRegularizationnon-search1337+147 / −1377.5729.30.4780.9707.40.0
9GainRatioInformation theorynon-search1320+156 / −1618.5327.30.7180.9720.10.0
10(R)ReliefFDistancenon-search1319+151 / −1548.2627.30.6820.9882.80.0
11SBEBackward searchsearch1286+152 / −1889.4432.50.1950.5803,601.361.1
12LOCOLeave-one-outsearch1269+107 / −1219.0132.50.2940.719346.60.0
13SFSForward searchsearch1259+118 / −1498.5729.40.4100.7092,046.216.7
14RandomBaseline1000+59 / −12012.1044.3-0.0060.4300.00.0
15LaplacianScoreDistancenon-search946+130 / −33312.9448.90.4990.8490.20.0
diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/pareto.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..1e462670922f3d7d86ecbc92ef7824f06d9fba50 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,37.8306,28.6344,12,True +MarkovBlanket,36.6063,12.908,12,True +CART,33.3076,10.5845,12,True +MI,32.5174,23.4943,12,False +mRMR,32.2874,8.5448,12,True +ElasticNet,30.4317,4484.8919,12,False +(R)ReliefF,29.1934,30686.1541,12,False +GainRatio,25.4813,3.4612,12,True +F-Test,25.1296,0.6541,12,True +Lasso,24.9532,6247.4428,12,False +SFS,21.3552,27637.3213,12,False +LOCO,16.9767,9859.2696,12,False +SBE,16.4981,49667.721,12,False +Random,0.0,0.004,12,True +LaplacianScore,-10.3927,4.2084,12,False diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/pareto.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..ac08e8122e75b38cee33cd4a3f05b9f8c7a29f2b --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 1
Hover or tap a method for its values.
10-210-110010110210310410505101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMarkovBlanketCARTMImRMRElasticNet(R)ReliefFGainRatioF-TestLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..b0748f39c1c561e843492908411521431722ddc2 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_0/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,CART,Non-search,Tree,1543.0,1454.0,1695.0,4.99,71.5,18.7,,0.536,0.827,0.0,0.0,12,36 +2,MarkovBlanket,Search,Backward search,1520.0,1416.0,1673.0,5.03,71.2,15.6,,0.473,0.491,0.2,0.0,12,36 +3,RFImportance,Non-search,Tree,1499.0,1400.0,1633.0,5.46,68.2,14.3,,0.68,0.96,0.1,0.0,12,36 +4,MI,Non-search,Information theory,1460.0,1375.0,1563.0,6.07,63.8,24.9,,0.734,0.988,0.2,0.0,12,36 +5,ElasticNet,Non-search,Regularization,1391.0,1257.0,1578.0,6.88,58.0,24.5,,0.585,0.975,19.1,0.0,12,36 +6,mRMR,Non-search,Information theory,1387.0,1284.0,1509.0,7.24,55.5,21.8,,0.615,0.958,0.3,0.0,12,36 +7,F-Test,Non-search,Correlation,1382.0,1295.0,1502.0,7.92,50.6,32.3,,0.758,1.0,0.0,0.0,12,36 +8,Lasso,Non-search,Regularization,1337.0,1200.0,1484.0,7.57,53.1,29.3,,0.478,0.97,7.4,0.0,12,36 +9,GainRatio,Non-search,Information theory,1320.0,1159.0,1476.0,8.53,46.2,27.3,,0.718,0.972,0.1,0.0,12,36 +10,(R)ReliefF,Non-search,Distance,1319.0,1165.0,1470.0,8.26,48.1,27.3,,0.682,0.988,2.8,0.0,12,36 +11,SBE,Search,Backward search,1286.0,1098.0,1438.0,9.44,39.7,32.5,,0.195,0.58,3601.3,61.1,12,36 +12,LOCO,Search,Leave-one-out,1269.0,1148.0,1376.0,9.01,42.8,32.5,,0.294,0.719,346.6,0.0,12,36 +13,SFS,Search,Forward search,1259.0,1110.0,1377.0,8.57,45.9,29.4,,0.41,0.709,2046.2,16.7,12,36 +14,Random,Baseline,Baseline,1000.0,880.0,1059.0,12.1,20.7,44.3,,-0.006,0.43,0.0,0.0,12,36 +15,LaplacianScore,Non-search,Distance,946.0,613.0,1076.0,12.94,14.7,48.9,,0.499,0.849,0.2,0.0,12,36 diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/elo_explorer.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..47b7075b0eeabbce0133833fd145503c2ae36c94 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 2
Hover or tap a method for its values.
60080010001200140016001800EloRandomLaplacianScoreSBESFSLOCOGainRatioF-Test(R)ReliefFLassoElasticNetmRMRMarkovBlanketMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/leaderboard_table.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..9266c568602dc793be30881377d0a49e19863c34 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 2
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1631+148 / −724.188.30.6910.8740.20.0
🥈2CARTTreenon-search1568+197 / −1245.1420.60.4600.6930.00.0
🥉3MIInformation theorynon-search1547+177 / −1215.3621.10.7250.9780.20.0
4MarkovBlanketBackward searchsearch1522+144 / −725.7921.20.4630.4960.20.0
5mRMRInformation theorynon-search1463+166 / −1256.9626.10.6780.8740.50.0
6ElasticNetRegularizationnon-search1458+179 / −1296.4423.40.6020.95919.20.0
7LassoRegularizationnon-search1453+202 / −1547.2223.30.5080.9157.50.0
8(R)ReliefFDistancenon-search1420+131 / −1107.8928.30.6950.9662.70.0
9F-TestCorrelationnon-search1415+153 / −1257.6429.40.7850.9860.00.0
10GainRatioInformation theorynon-search1354+129 / −1368.5427.90.6970.9760.10.0
11LOCOLeave-one-outsearch1270+135 / −1239.4433.70.2060.654419.50.0
12SFSForward searchsearch1243+84 / −1369.7532.30.3240.6543,600.141.7
13SBEBackward searchsearch1216+120 / −19910.8334.90.1390.5923,601.058.3
14LaplacianScoreDistancenon-search1044+176 / −43312.3148.60.5570.8430.20.0
15RandomBaseline1000+67 / −15612.5043.10.0020.4850.00.0
diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/pareto.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..50f52197d1ef79c974c521b5a586700ea50828e1 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,38.023,28.8109,12,True +MI,34.9015,23.3656,12,True +MarkovBlanket,33.8074,12.7536,12,True +CART,32.75,10.5308,12,True +mRMR,26.5395,14.9,12,False +ElasticNet,26.5125,4402.0539,12,False +GainRatio,26.0771,3.453,12,True +(R)ReliefF,25.24,30685.1988,12,False +F-Test,23.3784,0.5756,12,True +Lasso,23.0686,6208.3152,12,False +SFS,20.6301,42768.7575,12,False +LOCO,18.7287,9991.7859,12,False +SBE,12.855,47644.9092,12,False +Random,0.0,0.0025,12,True +LaplacianScore,-38.4849,4.2366,12,False diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/pareto.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..38f0a73db608c7d85e5cf7952d157a02f6fb4bfb --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 2
Hover or tap a method for its values.
10-210-110010110210310410505101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketCARTmRMRElasticNetGainRatio(R)ReliefFF-TestLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..9d18d1ff89e646b4bc3243dd2ad6826370ec73e9 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_1/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1631.0,1559.0,1779.0,4.18,77.3,8.3,,0.691,0.874,0.2,0.0,12,36 +2,CART,Non-search,Tree,1568.0,1444.0,1765.0,5.14,70.4,20.6,,0.46,0.693,0.0,0.0,12,36 +3,MI,Non-search,Information theory,1547.0,1426.0,1724.0,5.36,68.8,21.1,,0.725,0.978,0.2,0.0,12,36 +4,MarkovBlanket,Search,Backward search,1522.0,1450.0,1666.0,5.79,65.8,21.2,,0.463,0.496,0.2,0.0,12,36 +5,mRMR,Non-search,Information theory,1463.0,1338.0,1629.0,6.96,57.4,26.1,,0.678,0.874,0.5,0.0,12,36 +6,ElasticNet,Non-search,Regularization,1458.0,1329.0,1637.0,6.44,61.1,23.4,,0.602,0.959,19.2,0.0,12,36 +7,Lasso,Non-search,Regularization,1453.0,1299.0,1655.0,7.22,55.6,23.3,,0.508,0.915,7.5,0.0,12,36 +8,(R)ReliefF,Non-search,Distance,1420.0,1310.0,1551.0,7.89,50.8,28.3,,0.695,0.966,2.7,0.0,12,36 +9,F-Test,Non-search,Correlation,1415.0,1290.0,1568.0,7.64,52.6,29.4,,0.785,0.986,0.0,0.0,12,36 +10,GainRatio,Non-search,Information theory,1354.0,1218.0,1483.0,8.54,46.1,27.9,,0.697,0.976,0.1,0.0,12,36 +11,LOCO,Search,Leave-one-out,1270.0,1147.0,1405.0,9.44,39.7,33.7,,0.206,0.654,419.5,0.0,12,36 +12,SFS,Search,Forward search,1243.0,1107.0,1327.0,9.75,37.5,32.3,,0.324,0.654,3600.1,41.7,12,36 +13,SBE,Search,Backward search,1216.0,1017.0,1336.0,10.83,29.8,34.9,,0.139,0.592,3601.0,58.3,12,36 +14,LaplacianScore,Non-search,Distance,1044.0,611.0,1220.0,12.31,19.2,48.6,,0.557,0.843,0.2,0.0,12,36 +15,Random,Baseline,Baseline,1000.0,844.0,1067.0,12.5,17.9,43.1,,0.002,0.485,0.0,0.0,12,36 diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/elo_explorer.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..ddd231bc4c1a52a8e69f53db5a178c4409948706 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 3
Hover or tap a method for its values.
4006008001000120014001600EloLaplacianScoreSFSRandomLOCOSBEGainRatioF-TestLassoMarkovBlanketmRMRElasticNet(R)ReliefFCARTMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/leaderboard_table.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..7b4fea81cfe700955ff7f1e47613e398324272d6 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 3
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1402+144 / −915.1113.60.7050.8080.10.0
🥈2MIInformation theorynon-search1310+161 / −1195.5616.50.7220.9120.20.0
🥉3CARTTreenon-search1296+157 / −1125.8619.20.4300.6380.00.0
4(R)ReliefFDistancenon-search1277+126 / −1106.7518.50.7060.8932.70.0
5ElasticNetRegularizationnon-search1263+182 / −1386.1416.70.5720.88619.10.0
6mRMRInformation theorynon-search1240+149 / −1107.0818.50.6650.8150.70.0
7MarkovBlanketBackward searchsearch1226+123 / −766.9421.30.4780.4870.20.0
8LassoRegularizationnon-search1212+173 / −1536.4418.70.5180.8057.90.0
9F-TestCorrelationnon-search1208+139 / −1137.3322.10.7490.9390.00.0
10GainRatioInformation theorynon-search1132+107 / −898.4422.10.7320.8960.10.0
11SBEBackward searchsearch1051+135 / −13710.0327.30.1410.5073,600.252.8
12LOCOLeave-one-outsearch1026+143 / −16110.1729.10.1430.618321.611.1
13RandomBaseline1000+131 / −17910.9737.0-0.0010.5140.00.0
14SFSForward searchsearch944+67 / −8910.6128.50.3260.6373,600.052.8
15LaplacianScoreDistancenon-search866+164 / −50112.5644.80.6020.8240.20.0
diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/pareto.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..3b6849261a8b9cb008f7e0ae05f4ca8c4336b880 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,29.9814,29.5699,12,True +MI,25.9013,23.9206,12,True +mRMR,25.5658,27.8089,12,False +(R)ReliefF,25.4668,30686.2427,12,False +ElasticNet,19.8291,4360.3769,12,False +CART,19.4204,10.5872,12,True +GainRatio,17.9495,3.4863,12,True +MarkovBlanket,17.3307,12.5374,12,False +F-Test,15.0609,0.6125,12,True +Lasso,10.374,6215.8205,12,False +SFS,4.0995,48688.0763,12,False +SBE,1.0434,42467.9433,12,False +Random,0.0,0.0063,12,True +LOCO,-2.4851,9805.9976,12,False +LaplacianScore,-63.2183,4.0643,12,False diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/pareto.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..5a814e691ea7e6dbb6969c70e052d1e526fbdb84 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 3
Hover or tap a method for its values.
10-210-1100101102103104105051015202530Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMImRMR(R)ReliefFElasticNetCARTGainRatioMarkovBlanketF-TestLassoSFSSBERandomLOCODominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..17f890517f198fd410b99ee5003ebee482fee69d --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_2/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1402.0,1311.0,1546.0,5.11,70.6,13.6,,0.705,0.808,0.1,0.0,12,36 +2,MI,Non-search,Information theory,1310.0,1191.0,1471.0,5.56,67.5,16.5,,0.722,0.912,0.2,0.0,12,36 +3,CART,Non-search,Tree,1296.0,1184.0,1453.0,5.86,65.3,19.2,,0.43,0.638,0.0,0.0,12,36 +4,(R)ReliefF,Non-search,Distance,1277.0,1167.0,1403.0,6.75,58.9,18.5,,0.706,0.893,2.7,0.0,12,36 +5,ElasticNet,Non-search,Regularization,1263.0,1125.0,1445.0,6.14,63.3,16.7,,0.572,0.886,19.1,0.0,12,36 +6,mRMR,Non-search,Information theory,1240.0,1130.0,1389.0,7.08,56.6,18.5,,0.665,0.815,0.7,0.0,12,36 +7,MarkovBlanket,Search,Backward search,1226.0,1150.0,1349.0,6.94,57.5,21.3,,0.478,0.487,0.2,0.0,12,36 +8,Lasso,Non-search,Regularization,1212.0,1059.0,1385.0,6.44,61.1,18.7,,0.518,0.805,7.9,0.0,12,36 +9,F-Test,Non-search,Correlation,1208.0,1095.0,1347.0,7.33,54.8,22.1,,0.749,0.939,0.0,0.0,12,36 +10,GainRatio,Non-search,Information theory,1132.0,1043.0,1239.0,8.44,46.8,22.1,,0.732,0.896,0.1,0.0,12,36 +11,SBE,Search,Backward search,1051.0,914.0,1186.0,10.03,35.5,27.3,,0.141,0.507,3600.2,52.8,12,36 +12,LOCO,Search,Leave-one-out,1026.0,865.0,1169.0,10.17,34.5,29.1,,0.143,0.618,321.6,11.1,12,36 +13,Random,Baseline,Baseline,1000.0,821.0,1131.0,10.97,28.8,37.0,,-0.001,0.514,0.0,0.0,12,36 +14,SFS,Search,Forward search,944.0,855.0,1011.0,10.61,31.4,28.5,,0.326,0.637,3600.0,52.8,12,36 +15,LaplacianScore,Non-search,Distance,866.0,365.0,1030.0,12.56,17.5,44.8,,0.602,0.824,0.2,0.0,12,36 diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/elo_explorer.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..4bfbde0425a2f638d96b01a636792ff9b782fd45 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 4
Hover or tap a method for its values.
800100012001400160018002000EloRandomLaplacianScoreSFSLOCOSBEF-TestmRMRGainRatioLassoElasticNetCARTMarkovBlanket(R)ReliefFMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/leaderboard_table.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..69bef3b282e71b034b78a95da3d4a75e7a815313 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 4
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1635+168 / −933.447.10.7050.6850.10.0
🥈2MIInformation theorynon-search1446+146 / −935.3911.20.7040.7970.20.0
🥉3(R)ReliefFDistancenon-search1345+114 / −887.3315.10.6640.7622.70.0
4MarkovBlanketBackward searchsearch1314+124 / −986.8217.20.5020.4790.20.0
5CARTTreenon-search1306+166 / −1376.3316.80.3800.5790.00.0
6ElasticNetRegularizationnon-search1287+111 / −907.2816.60.5470.78219.50.0
7LassoRegularizationnon-search1246+154 / −1377.3616.10.4550.7098.50.0
8GainRatioInformation theorynon-search1243+131 / −1158.3217.60.7330.7530.10.0
9mRMRInformation theorynon-search1205+103 / −1127.8917.30.6750.6920.98.3
10F-TestCorrelationnon-search1187+131 / −1348.0317.80.6760.8140.00.0
11SBEBackward searchsearch1182+147 / −1499.1522.50.1060.5103,219.744.4
12LOCOLeave-one-outsearch1092+84 / −1049.5824.30.1490.575316.516.7
13SFSForward searchsearch1066+117 / −18010.0623.90.2670.5663,600.961.1
14RandomBaseline1000+141 / −22010.8831.30.0020.4860.00.0
15LaplacianScoreDistancenon-search1000+149 / −23412.1437.80.6630.7330.20.0
diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/pareto.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..b533c20baad3fdfea2f23f078b477ca11ab849b4 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,28.3971,28.9109,12,True +MI,24.6469,23.3372,12,True +mRMR,21.3261,56.5337,12,False +CART,20.5996,10.4669,12,True +(R)ReliefF,20.5827,30686.3556,12,False +MarkovBlanket,19.6059,12.2262,12,False +ElasticNet,18.1233,4324.3577,12,False +GainRatio,15.8024,3.4734,12,True +F-Test,14.5835,0.6041,12,True +Lasso,13.8789,6202.8464,12,False +SFS,10.896,52703.8375,12,False +LOCO,9.9324,9702.1677,12,False +SBE,8.7515,35259.7017,12,False +Random,0.0,0.0021,12,True +LaplacianScore,-24.9809,4.3517,12,False diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/pareto.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..901c874b614dbd8a983c1ea77772749d33c373f2 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 4
Hover or tap a method for its values.
10-310-210-11001011021031041050510152025Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMImRMRCART(R)ReliefFMarkovBlanketElasticNetGainRatioF-TestLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..89db3c8ffa73681cae8f3ce18adbf2d4b95833fd --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_3/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1635.0,1542.0,1803.0,3.44,82.5,7.1,,0.705,0.685,0.1,0.0,12,36 +2,MI,Non-search,Information theory,1446.0,1353.0,1592.0,5.39,68.6,11.2,,0.704,0.797,0.2,0.0,12,36 +3,(R)ReliefF,Non-search,Distance,1345.0,1257.0,1459.0,7.33,54.8,15.1,,0.664,0.762,2.7,0.0,12,36 +4,MarkovBlanket,Search,Backward search,1314.0,1216.0,1438.0,6.82,58.4,17.2,,0.502,0.479,0.2,0.0,12,36 +5,CART,Non-search,Tree,1306.0,1169.0,1472.0,6.33,61.9,16.8,,0.38,0.579,0.0,0.0,12,36 +6,ElasticNet,Non-search,Regularization,1287.0,1197.0,1398.0,7.28,55.2,16.6,,0.547,0.782,19.5,0.0,12,36 +7,Lasso,Non-search,Regularization,1246.0,1109.0,1400.0,7.36,54.6,16.1,,0.455,0.709,8.5,0.0,12,36 +8,GainRatio,Non-search,Information theory,1243.0,1128.0,1374.0,8.32,47.7,17.6,,0.733,0.753,0.1,0.0,12,36 +9,mRMR,Non-search,Information theory,1205.0,1093.0,1308.0,7.89,50.8,17.3,,0.675,0.692,0.9,8.3,12,36 +10,F-Test,Non-search,Correlation,1187.0,1053.0,1318.0,8.03,49.8,17.8,,0.676,0.814,0.0,0.0,12,36 +11,SBE,Search,Backward search,1182.0,1033.0,1329.0,9.15,41.8,22.5,,0.106,0.51,3219.7,44.4,12,36 +12,LOCO,Search,Leave-one-out,1092.0,988.0,1176.0,9.58,38.7,24.3,,0.149,0.575,316.5,16.7,12,36 +13,SFS,Search,Forward search,1066.0,886.0,1183.0,10.06,35.3,23.9,,0.267,0.566,3600.9,61.1,12,36 +14,Random,Baseline,Baseline,1000.0,780.0,1141.0,10.88,29.5,31.3,,0.002,0.486,0.0,0.0,12,36 +15,LaplacianScore,Non-search,Distance,1000.0,766.0,1149.0,12.14,20.4,37.8,,0.663,0.733,0.2,0.0,12,36 diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/elo_explorer.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..710fe846b4cece5a943a47e6606047def21ea05d --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 5
Hover or tap a method for its values.
9001000110012001300140015001600EloRandomSFSLaplacianScoreLOCOSBEmRMRLassoElasticNetGainRatioMarkovBlanket(R)ReliefFF-TestMICARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/leaderboard_table.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..deebd7a77cad71a31d21ce11771d79b0a58d0a8d --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 5
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1429+170 / −1184.657.40.6360.5530.10.0
🥈2CARTTreenon-search1289+172 / −1346.3313.20.3680.5180.00.0
🥉3MIInformation theorynon-search1284+103 / −806.6512.30.6310.6760.20.0
4(R)ReliefFDistancenon-search1226+128 / −1167.2513.50.6480.6532.70.0
5F-TestCorrelationnon-search1226+151 / −1287.2813.40.6230.7070.00.0
6MarkovBlanketBackward searchsearch1208+156 / −1297.3913.80.4000.4680.10.0
7GainRatioInformation theorynon-search1183+99 / −1157.5312.30.6010.6680.10.0
8ElasticNetRegularizationnon-search1174+110 / −1047.8313.50.4280.63919.20.0
9LassoRegularizationnon-search1170+117 / −1267.5314.50.3480.6107.80.0
10mRMRInformation theorynon-search1119+91 / −1029.0614.10.5760.6191.08.3
11SBEBackward searchsearch1097+141 / −1619.0818.60.0750.5141,744.341.7
12LOCOLeave-one-outsearch1062+85 / −1099.6919.30.1130.518319.716.7
13LaplacianScoreDistancenon-search1029+147 / −17610.3122.30.6080.6630.20.0
14SFSForward searchsearch1010+90 / −1369.3317.80.2910.5163,600.561.1
15RandomBaseline1000+121 / −13910.0818.50.0040.5040.00.0
diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/pareto.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..b465ae964542bc12f723a7827d68dd969959a6d3 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,14.1492,29.2645,12,True +GainRatio,10.61,3.482,12,True +MI,10.4437,23.7282,12,False +MarkovBlanket,8.8574,11.8984,12,False +(R)ReliefF,8.7418,30687.3381,12,False +F-Test,8.6209,0.6297,12,True +mRMR,8.4594,96.1753,12,False +ElasticNet,8.4,4364.6248,12,False +Lasso,6.7545,6299.8114,12,False +CART,5.0364,10.5301,12,False +SBE,1.8326,22877.8198,12,False +SFS,0.6738,55470.8243,12,False +Random,0.0,0.0022,12,True +LOCO,-0.6738,9624.4906,12,False +LaplacianScore,-5.2241,4.4444,12,False diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/pareto.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..8b805e15fdc56c391be5bb9d34e21023c048996e --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, TabICLv2, budget 5
Hover or tap a method for its values.
10-310-210-110010110210310410502468101214Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceGainRatioMIMarkovBlanket(R)ReliefFF-TestmRMRElasticNetLassoCARTSBESFSRandomLOCODominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..d625f629856082d44b771437b0b687268711f402 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_4/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1429.0,1311.0,1599.0,4.65,73.9,7.4,,0.636,0.553,0.1,0.0,12,36 +2,CART,Non-search,Tree,1289.0,1155.0,1461.0,6.33,61.9,13.2,,0.368,0.518,0.0,0.0,12,36 +3,MI,Non-search,Information theory,1284.0,1204.0,1387.0,6.65,59.6,12.3,,0.631,0.676,0.2,0.0,12,36 +4,(R)ReliefF,Non-search,Distance,1226.0,1110.0,1354.0,7.25,55.4,13.5,,0.648,0.653,2.7,0.0,12,36 +5,F-Test,Non-search,Correlation,1226.0,1098.0,1377.0,7.28,55.2,13.4,,0.623,0.707,0.0,0.0,12,36 +6,MarkovBlanket,Search,Backward search,1208.0,1079.0,1364.0,7.39,54.4,13.8,,0.4,0.468,0.1,0.0,12,36 +7,GainRatio,Non-search,Information theory,1183.0,1068.0,1282.0,7.53,53.4,12.3,,0.601,0.668,0.1,0.0,12,36 +8,ElasticNet,Non-search,Regularization,1174.0,1070.0,1284.0,7.83,51.2,13.5,,0.428,0.639,19.2,0.0,12,36 +9,Lasso,Non-search,Regularization,1170.0,1044.0,1287.0,7.53,53.4,14.5,,0.348,0.61,7.8,0.0,12,36 +10,mRMR,Non-search,Information theory,1119.0,1017.0,1210.0,9.06,42.5,14.1,,0.576,0.619,1.0,8.3,12,36 +11,SBE,Search,Backward search,1097.0,936.0,1238.0,9.08,42.3,18.6,,0.075,0.514,1744.3,41.7,12,36 +12,LOCO,Search,Leave-one-out,1062.0,953.0,1147.0,9.69,37.9,19.3,,0.113,0.518,319.7,16.7,12,36 +13,LaplacianScore,Non-search,Distance,1029.0,853.0,1176.0,10.31,33.5,22.3,,0.608,0.663,0.2,0.0,12,36 +14,SFS,Search,Forward search,1010.0,874.0,1100.0,9.33,40.5,17.8,,0.291,0.516,3600.5,61.1,12,36 +15,Random,Baseline,Baseline,1000.0,861.0,1121.0,10.08,35.1,18.5,,0.004,0.504,0.0,0.0,12,36 diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/elo_explorer.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..34c7b4d074654d3e1f6cef323fefe63d0546d6f2 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: multiclass, TabICLv2, all budgets
Hover or tap a method for its values.
8001000120014001600EloLaplacianScoreRandomSFSLOCOSBEGainRatiomRMRLassoF-TestElasticNet(R)ReliefFMarkovBlanketCARTMIRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/leaderboard_table.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..85cffd82c1a9b9281f6bee80c3410d6b221fbbf5 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: multiclass, TabICLv2, all budgets
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1489+83 / −544.5710.20.6830.7760.10.0
🥈2MIInformation theorynon-search1388+95 / −725.8117.20.7030.8700.20.0
🥉3CARTTreenon-search1379+141 / −1065.7317.70.4350.6510.00.0
4MarkovBlanketBackward searchsearch1337+81 / −626.3917.80.4630.4840.20.0
5(R)ReliefFDistancenon-search1298+82 / −757.5020.50.6790.8522.70.0
6ElasticNetRegularizationnon-search1296+102 / −976.9118.90.5470.84819.10.0
7F-TestCorrelationnon-search1266+90 / −837.6423.00.7180.8890.00.0
8LassoRegularizationnon-search1265+110 / −1097.2220.40.4610.8027.40.0
9mRMRInformation theorynon-search1264+85 / −917.6419.60.6420.7920.53.3
10GainRatioInformation theorynon-search1230+86 / −1018.2721.40.6960.8530.10.0
11SBEBackward searchsearch1153+89 / −1049.7127.20.1260.5413,600.151.7
12LOCOLeave-one-outsearch1129+89 / −999.5827.80.1810.617330.18.9
13SFSForward searchsearch1092+65 / −919.6626.40.3310.6163,526.646.7
14RandomBaseline1000+72 / −9611.3134.80.0000.4840.00.0
15LaplacianScoreDistancenon-search979+120 / −19312.0540.50.5860.7820.20.0
diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/pareto.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..9b00201e5e1af7beb3b6b1cc745513b6b7c4806c --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,35.7401,29.0381,12,True +MI,31.5315,23.5692,12,True +MarkovBlanket,31.3202,12.4647,12,True +CART,29.6499,10.5399,12,True +mRMR,29.6029,40.7925,12,False +(R)ReliefF,26.6882,30686.2579,12,False +ElasticNet,26.1045,4387.2611,12,False +GainRatio,24.4496,3.4712,12,True +F-Test,23.0192,0.6152,12,True +Lasso,21.8858,6234.8473,12,False +SFS,19.072,45453.7634,12,False +LOCO,15.337,9796.7423,12,False +SBE,13.5125,39583.619,12,False +Random,0.0,0.0034,12,True +LaplacianScore,-15.7051,4.2611,12,False diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/pareto.html b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..2157e33ec3d28ead84d2a357cf82bd7bcfda9862 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: multiclass, TabICLv2, all budgets
Hover or tap a method for its values.
10-210-110010110210310410505101520253035Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMIMarkovBlanketCARTmRMR(R)ReliefFElasticNetGainRatioF-TestLassoSFSLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/website_leaderboard.csv b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..d90bdfda6af88b5b419d813ab12d84f05eeee207 --- /dev/null +++ b/data/subsets/tasks_multiclass/models_tabiclv2/budgets_all/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1489.0,1435.0,1572.0,4.57,74.5,10.2,,0.683,0.776,0.1,0.0,12,180 +2,MI,Non-search,Information theory,1388.0,1316.0,1483.0,5.81,65.7,17.2,,0.703,0.87,0.2,0.0,12,180 +3,CART,Non-search,Tree,1379.0,1273.0,1520.0,5.73,66.2,17.7,,0.435,0.651,0.0,0.0,12,180 +4,MarkovBlanket,Search,Backward search,1337.0,1275.0,1418.0,6.39,61.5,17.8,,0.463,0.484,0.2,0.0,12,180 +5,(R)ReliefF,Non-search,Distance,1298.0,1223.0,1380.0,7.5,53.6,20.5,,0.679,0.852,2.7,0.0,12,180 +6,ElasticNet,Non-search,Regularization,1296.0,1199.0,1398.0,6.91,57.8,18.9,,0.547,0.848,19.1,0.0,12,180 +7,F-Test,Non-search,Correlation,1266.0,1183.0,1356.0,7.64,52.6,23.0,,0.718,0.889,0.0,0.0,12,180 +8,Lasso,Non-search,Regularization,1265.0,1156.0,1375.0,7.22,55.5,20.4,,0.461,0.802,7.4,0.0,12,180 +9,mRMR,Non-search,Information theory,1264.0,1173.0,1349.0,7.64,52.5,19.6,,0.642,0.792,0.5,3.3,12,180 +10,GainRatio,Non-search,Information theory,1230.0,1129.0,1316.0,8.27,48.1,21.4,,0.696,0.853,0.1,0.0,12,180 +11,SBE,Search,Backward search,1153.0,1049.0,1242.0,9.71,37.8,27.2,,0.126,0.541,3600.1,51.7,12,180 +12,LOCO,Search,Leave-one-out,1129.0,1030.0,1218.0,9.58,38.7,27.8,,0.181,0.617,330.1,8.9,12,180 +13,SFS,Search,Forward search,1092.0,1001.0,1157.0,9.66,38.1,26.4,,0.331,0.616,3526.6,46.7,12,180 +14,Random,Baseline,Baseline,1000.0,904.0,1072.0,11.31,26.4,34.8,,0.0,0.484,0.0,0.0,12,180 +15,LaplacianScore,Non-search,Distance,979.0,786.0,1099.0,12.05,21.1,40.5,,0.586,0.782,0.2,0.0,12,180 diff --git a/data/subsets/tasks_regression/models_all/budgets_0/pareto.html b/data/subsets/tasks_regression/models_all/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..3bfd40407a7c7be5354820afa3b6488c4f102716 --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, all models, budget 1
Hover or tap a method for its values.
10-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIElasticNetLassoF-TestMarkovBlanketSFSSBEGainRatiomRMR(R)ReliefFLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_all/budgets_0/per_model.csv b/data/subsets/tasks_regression/models_all/budgets_0/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..ffba878a89f151bb6e3be00b780737894d0fb406 --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_0/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1348.0,1512.0,1,1357.0,1,1246.0,2,1327.0,1,1361.0,1 +2,ElasticNet,Non-search,Regularization,1314.0,1435.0,3,1348.0,2,1257.0,1,1318.0,2,1261.0,4 +3,CART,Non-search,Tree,1313.0,1469.0,2,1309.0,7,1204.0,7,1316.0,3,1324.0,2 +4,SFS,Search,Forward search,1284.0,1413.0,4,1327.0,3,1218.0,6,1223.0,7,1288.0,3 +5,MarkovBlanket,Search,Backward search,1273.0,1368.0,6,1324.0,5,1192.0,8,1296.0,4,1234.0,5 +6,Lasso,Non-search,Regularization,1269.0,1378.0,5,1294.0,8,1218.0,4,1264.0,5,1234.0,6 +7,MI,Non-search,Information theory,1242.0,1332.0,7,1316.0,6,1218.0,5,1262.0,6,1127.0,11 +8,F-Test,Non-search,Correlation,1238.0,1309.0,8,1325.0,4,1220.0,3,1219.0,8,1163.0,8 +9,(R)ReliefF,Non-search,Distance,1156.0,1212.0,11,1179.0,11,1091.0,12,1174.0,10,1158.0,9 +10,SBE,Search,Backward search,1156.0,1305.0,9,1075.0,13,1096.0,11,1174.0,11,1158.0,10 +11,mRMR,Non-search,Information theory,1153.0,1188.0,12,1229.0,10,1147.0,10,1202.0,9,1026.0,13 +12,GainRatio,Non-search,Information theory,1146.0,1175.0,13,1247.0,9,1154.0,9,1096.0,12,1083.0,12 +13,LOCO,Search,Leave-one-out,1126.0,1224.0,10,1123.0,12,1060.0,13,1073.0,13,1178.0,7 +14,Random,Baseline,Baseline,1000.0,1000.0,15,1000.0,14,1000.0,14,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,966.0,1048.0,14,821.0,15,994.0,15,928.0,15,994.0,15 diff --git a/data/subsets/tasks_regression/models_all/budgets_0/per_model_table.html b/data/subsets/tasks_regression/models_all/budgets_0/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..d86e0ef728815ae3ccfcf38c10bdaa93ea8e53a2 --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_0/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, all models, budget 1 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1348🥇1512🥇1357🥈1246🥇1327🥇1361
🥈2ElasticNet1314🥉1435🥈1348🥇1257🥈13181261
🥉3CART1313🥈146913091204🥉1316🥈1324
4SFS12841413🥉132712181223🥉1288
5MarkovBlanket127313681324119212961234
6Lasso126913781294121812641234
7MI124213321316121812621127
8F-Test123813091325🥉122012191163
9(R)ReliefF115612121179109111741158
10SBE115613051075109611741158
11mRMR115311881229114712021026
12GainRatio114611751247115410961083
13LOCO112612241123106010731178
14Random100010001000100010001000
15LaplacianScore9661048821994928994
diff --git a/data/subsets/tasks_regression/models_all/budgets_1/pareto.html b/data/subsets/tasks_regression/models_all/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..88eba6445609587fb04693acb12d7b8d1c15404e --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, all models, budget 2
Hover or tap a method for its values.
10-210-110010110210310402468101214Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIElasticNetLassoSFSF-TestGainRatioMarkovBlanketSBE(R)ReliefFmRMRLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_all/budgets_1/per_model.csv b/data/subsets/tasks_regression/models_all/budgets_1/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..7c195c613e1988fea945c2d495bcb05a9943784a --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_1/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1439.0,1425.0,1,1463.0,1,1508.0,1,1369.0,1,1476.0,2 +2,CART,Non-search,Tree,1423.0,1401.0,2,1432.0,2,1476.0,2,1345.0,2,1508.0,1 +3,ElasticNet,Non-search,Regularization,1342.0,1398.0,3,1403.0,4,1294.0,9,1278.0,5,1380.0,4 +4,Lasso,Non-search,Regularization,1332.0,1343.0,5,1420.0,3,1289.0,10,1262.0,6,1391.0,3 +5,MarkovBlanket,Search,Backward search,1326.0,1309.0,6,1398.0,5,1351.0,5,1304.0,3,1308.0,8 +6,MI,Non-search,Information theory,1306.0,1270.0,8,1368.0,6,1409.0,3,1201.0,9,1318.0,7 +7,SBE,Search,Backward search,1283.0,1306.0,7,1203.0,12,1340.0,6,1299.0,4,1297.0,9 +8,F-Test,Non-search,Correlation,1268.0,1206.0,12,1335.0,7,1304.0,7,1186.0,10,1340.0,6 +9,SFS,Search,Forward search,1259.0,1358.0,4,1220.0,10,1258.0,11,1226.0,7,1260.0,10 +10,GainRatio,Non-search,Information theory,1254.0,1227.0,10,1313.0,8,1303.0,8,1209.0,8,1249.0,11 +11,LOCO,Search,Leave-one-out,1239.0,1239.0,9,1257.0,9,1216.0,12,1134.0,12,1377.0,5 +12,(R)ReliefF,Non-search,Distance,1229.0,1208.0,11,1214.0,11,1351.0,4,1160.0,11,1239.0,12 +13,mRMR,Non-search,Information theory,1148.0,1160.0,13,1176.0,13,1156.0,14,1120.0,13,1148.0,13 +14,LaplacianScore,Non-search,Distance,1076.0,1090.0,14,1082.0,14,1205.0,13,965.0,15,1033.0,14 +15,Random,Baseline,Baseline,1000.0,1000.0,15,1000.0,15,1000.0,15,1000.0,14,1000.0,15 diff --git a/data/subsets/tasks_regression/models_all/budgets_1/per_model_table.html b/data/subsets/tasks_regression/models_all/budgets_1/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..ae8d7dfe606b775ac58c9b120b818cd3154356e8 --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_1/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, all models, budget 2 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1439🥇1425🥇1463🥇1508🥇1369🥈1476
🥈2CART1423🥈1401🥈1432🥈1476🥈1345🥇1508
🥉3ElasticNet1342🥉13981403129412781380
4Lasso13321343🥉142012891262🥉1391
5MarkovBlanket1326130913981351🥉13041308
6MI130612701368🥉140912011318
7SBE128313061203134012991297
8F-Test126812061335130411861340
9SFS125913581220125812261260
10GainRatio125412271313130312091249
11LOCO123912391257121611341377
12(R)ReliefF122912081214135111601239
13mRMR114811601176115611201148
14LaplacianScore10761090108212059651033
15Random100010001000100010001000
diff --git a/data/subsets/tasks_regression/models_all/budgets_2/pareto.html b/data/subsets/tasks_regression/models_all/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..b69de1321ceda335550c5bcf844ce9d0038ba923 --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, all models, budget 3
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMI(R)ReliefFLOCOLassoElasticNetF-TestMarkovBlanketSFSSBEGainRatiomRMRRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_all/budgets_2/per_model.csv b/data/subsets/tasks_regression/models_all/budgets_2/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..efa0d7e982ceee18cf2de4464dbf1982977b1e4c --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_2/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1518.0,1536.0,2,1498.0,1,1574.0,2,1490.0,1,1587.0,1 +2,CART,Non-search,Tree,1508.0,1558.0,1,1396.0,2,1641.0,1,1490.0,2,1554.0,2 +3,(R)ReliefF,Non-search,Distance,1321.0,1315.0,8,1222.0,9,1359.0,4,1376.0,3,1402.0,4 +4,Lasso,Non-search,Regularization,1308.0,1322.0,7,1347.0,4,1297.0,10,1280.0,7,1356.0,5 +5,MI,Non-search,Information theory,1294.0,1345.0,5,1291.0,5,1343.0,6,1261.0,9,1290.0,8 +6,LOCO,Search,Leave-one-out,1291.0,1368.0,4,1052.0,12,1397.0,3,1283.0,6,1407.0,3 +7,ElasticNet,Non-search,Regularization,1285.0,1253.0,11,1358.0,3,1297.0,11,1280.0,8,1301.0,7 +8,MarkovBlanket,Search,Backward search,1276.0,1331.0,6,1206.0,10,1295.0,12,1326.0,4,1279.0,9 +9,SBE,Search,Backward search,1265.0,1400.0,3,1027.0,14,1327.0,8,1299.0,5,1312.0,6 +10,GainRatio,Non-search,Information theory,1256.0,1281.0,10,1259.0,6,1330.0,7,1243.0,10,1221.0,12 +11,SFS,Search,Forward search,1241.0,1304.0,9,1238.0,8,1257.0,13,1229.0,11,1224.0,11 +12,F-Test,Non-search,Correlation,1206.0,1178.0,13,1254.0,7,1348.0,5,1110.0,13,1172.0,13 +13,mRMR,Non-search,Information theory,1194.0,1232.0,12,1028.0,13,1303.0,9,1199.0,12,1243.0,10 +14,LaplacianScore,Non-search,Distance,1036.0,1132.0,14,1077.0,11,1044.0,14,912.0,15,991.0,15 +15,Random,Baseline,Baseline,1000.0,1000.0,15,1000.0,15,1000.0,15,1000.0,14,1000.0,14 diff --git a/data/subsets/tasks_regression/models_all/budgets_2/per_model_table.html b/data/subsets/tasks_regression/models_all/budgets_2/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..3c5e292af573f731fd61c621faa47ba9f5b4a1f0 --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_2/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, all models, budget 3 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1518🥈1536🥇1498🥈1574🥇1490🥇1587
🥈2CART1508🥇1558🥈1396🥇1641🥈1490🥈1554
🥉3(R)ReliefF1321131512221359🥉13761402
4Lasso130813221347129712801356
5MI129413451291134312611290
6LOCO129113681052🥉13971283🥉1407
7ElasticNet12851253🥉1358129712801301
8MarkovBlanket127613311206129513261279
9SBE1265🥉14001027132712991312
10GainRatio125612811259133012431221
11SFS124113041238125712291224
12F-Test120611781254134811101172
13mRMR119412321028130311991243
14LaplacianScore1036113210771044912991
15Random100010001000100010001000
diff --git a/data/subsets/tasks_regression/models_all/budgets_3/pareto.html b/data/subsets/tasks_regression/models_all/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..b43802d3d8282b8e0f4797b87d648ee104e2eebd --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, all models, budget 4
Hover or tap a method for its values.
10-210-1100101102103104012345678Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCART(R)ReliefFMIF-TestLOCOLassoElasticNetGainRatioMarkovBlanketmRMRSFSSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_all/budgets_3/per_model.csv b/data/subsets/tasks_regression/models_all/budgets_3/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..0a130dd66b56a4db044f71f05f44f4ac9b22a81c --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_3/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1372.0,1411.0,1,1319.0,3,1417.0,2,1289.0,3,1471.0,1 +2,CART,Non-search,Tree,1360.0,1355.0,2,1313.0,4,1436.0,1,1329.0,2,1405.0,2 +3,(R)ReliefF,Non-search,Distance,1320.0,1228.0,6,1299.0,5,1387.0,3,1334.0,1,1386.0,3 +4,Lasso,Non-search,Regularization,1273.0,1269.0,5,1356.0,1,1252.0,6,1214.0,5,1306.0,5 +5,LOCO,Search,Leave-one-out,1257.0,1278.0,4,1126.0,11,1298.0,4,1273.0,4,1340.0,4 +6,ElasticNet,Non-search,Regularization,1229.0,1211.0,8,1347.0,2,1252.0,7,1176.0,7,1188.0,12 +7,MI,Non-search,Information theory,1219.0,1281.0,3,1258.0,6,1236.0,9,1129.0,11,1212.0,8 +8,F-Test,Non-search,Correlation,1200.0,1226.0,7,1214.0,7,1184.0,12,1175.0,8,1223.0,6 +9,SFS,Search,Forward search,1189.0,1185.0,10,1142.0,10,1246.0,8,1175.0,9,1218.0,7 +10,mRMR,Non-search,Information theory,1170.0,1141.0,12,1096.0,12,1219.0,10,1208.0,6,1199.0,10 +11,MarkovBlanket,Search,Backward search,1166.0,1095.0,13,1178.0,9,1262.0,5,1113.0,13,1196.0,11 +12,SBE,Search,Backward search,1159.0,1195.0,9,1066.0,13,1195.0,11,1170.0,10,1180.0,13 +13,GainRatio,Non-search,Information theory,1155.0,1148.0,11,1188.0,8,1133.0,13,1115.0,12,1204.0,9 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,15,1000.0,15,1000.0,14,1000.0,14 +15,LaplacianScore,Non-search,Distance,975.0,958.0,15,1066.0,14,1007.0,14,876.0,15,946.0,15 diff --git a/data/subsets/tasks_regression/models_all/budgets_3/per_model_table.html b/data/subsets/tasks_regression/models_all/budgets_3/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..c6c695b3dd29e9985272bd7ad4f2eb5d7b9e67f7 --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_3/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, all models, budget 4 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1372🥇1411🥉1319🥈1417🥉1289🥇1471
🥈2CART1360🥈13551313🥇1436🥈1329🥈1405
🥉3(R)ReliefF132012281299🥉1387🥇1334🥉1386
4Lasso12731269🥇1356125212141306
5LOCO125712781126129812731340
6ElasticNet12291211🥈1347125211761188
7MI1219🥉12811258123611291212
8F-Test120012261214118411751223
9SFS118911851142124611751218
10mRMR117011411096121912081199
11MarkovBlanket116610951178126211131196
12SBE115911951066119511701180
13GainRatio115511481188113311151204
14Random100010001000100010001000
15LaplacianScore97595810661007876946
diff --git a/data/subsets/tasks_regression/models_all/budgets_4/pareto.html b/data/subsets/tasks_regression/models_all/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..06c39932fe9537dc8da675a67c927ff8a12f91c6 --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, all models, budget 5
Hover or tap a method for its values.
10-210-110010110210310401234Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalCARTRFImportanceMI(R)ReliefFElasticNetLOCOMarkovBlanketF-TestLassoGainRatiomRMRSBESFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_all/budgets_4/per_model.csv b/data/subsets/tasks_regression/models_all/budgets_4/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..04d137072ac0d666e12c76a66fd6a9219dc2fc64 --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_4/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,(R)ReliefF,Non-search,Distance,1260.0,1169.0,6,1258.0,3,1331.0,1,1216.0,2,1364.0,1 +2,RFImportance,Non-search,Tree,1237.0,1257.0,1,1242.0,4,1232.0,2,1175.0,6,1308.0,2 +3,CART,Non-search,Tree,1220.0,1241.0,3,1194.0,6,1227.0,3,1185.0,3,1277.0,5 +4,ElasticNet,Non-search,Regularization,1216.0,1203.0,5,1282.0,2,1161.0,5,1167.0,7,1290.0,3 +5,Lasso,Non-search,Regularization,1216.0,1242.0,2,1294.0,1,1109.0,10,1178.0,5,1284.0,4 +6,MI,Non-search,Information theory,1197.0,1237.0,4,1224.0,5,1155.0,6,1232.0,1,1166.0,11 +7,LOCO,Search,Leave-one-out,1147.0,1169.0,7,1116.0,10,1095.0,12,1180.0,4,1191.0,9 +8,GainRatio,Non-search,Information theory,1130.0,1109.0,9,1149.0,8,1097.0,11,1104.0,9,1206.0,8 +9,MarkovBlanket,Search,Backward search,1121.0,1081.0,11,1070.0,11,1221.0,4,1101.0,10,1146.0,12 +10,mRMR,Non-search,Information theory,1120.0,1076.0,12,1164.0,7,1143.0,7,1049.0,12,1181.0,10 +11,SBE,Search,Backward search,1113.0,1159.0,8,1058.0,12,1125.0,9,1106.0,8,1131.0,13 +12,F-Test,Non-search,Correlation,1101.0,1084.0,10,1136.0,9,1006.0,13,1082.0,11,1206.0,7 +13,SFS,Search,Forward search,1073.0,1006.0,13,1036.0,14,1130.0,8,967.0,15,1226.0,6 +14,Random,Baseline,Baseline,1000.0,1000.0,14,1000.0,15,1000.0,14,1000.0,13,1000.0,15 +15,LaplacianScore,Non-search,Distance,989.0,971.0,15,1044.0,13,908.0,15,989.0,14,1025.0,14 diff --git a/data/subsets/tasks_regression/models_all/budgets_4/per_model_table.html b/data/subsets/tasks_regression/models_all/budgets_4/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..8c313712d48238f4beb772798ffff9f99e5bac0a --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_4/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, all models, budget 5 · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1(R)ReliefF12601169🥉1258🥇1331🥈1216🥇1364
🥈2RFImportance1237🥇12571242🥈12321175🥈1308
🥉3CART1220🥉12411194🥉1227🥉11851277
4ElasticNet12161203🥈128211611167🥉1290
5Lasso1216🥈1242🥇1294110911781284
6MI1197123712241155🥇12321166
7LOCO114711691116109511801191
8GainRatio113011091149109711041206
9MarkovBlanket112110811070122111011146
10mRMR112010761164114310491181
11SBE111311591058112511061131
12F-Test110110841136100610821206
13SFS10731006103611309671226
14Random100010001000100010001000
15LaplacianScore98997110449089891025
diff --git a/data/subsets/tasks_regression/models_all/budgets_all/pareto.html b/data/subsets/tasks_regression/models_all/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..4df5dbb3009cc8fdd657541073c295f2faa48e4e --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, all models, all budgets
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIElasticNetLassoF-TestMarkovBlanket(R)ReliefFSFSGainRatioLOCOSBEmRMRRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_all/budgets_all/per_model.csv b/data/subsets/tasks_regression/models_all/budgets_all/per_model.csv new file mode 100644 index 0000000000000000000000000000000000000000..cdc6441f4f53c3a02ad806702d1f97baf5e88017 --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_all/per_model.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo LightGBM,Rank LightGBM,Elo LinearModel,Rank LinearModel,Elo RandomForest,Rank RandomForest,Elo RealMLP,Rank RealMLP,Elo TabICLv2,Rank TabICLv2 +1,RFImportance,Non-search,Tree,1372.0,1411.0,1,1358.0,1,1372.0,1,1315.0,2,1422.0,1 +2,CART,Non-search,Tree,1354.0,1387.0,2,1316.0,4,1368.0,2,1318.0,1,1396.0,2 +3,Lasso,Non-search,Regularization,1273.0,1298.0,3,1330.0,3,1219.0,7,1230.0,5,1304.0,3 +4,ElasticNet,Non-search,Regularization,1270.0,1285.0,4,1336.0,2,1238.0,6,1234.0,4,1275.0,6 +5,(R)ReliefF,Non-search,Distance,1250.0,1217.0,10,1225.0,7,1283.0,3,1240.0,3,1297.0,4 +6,MI,Non-search,Information theory,1245.0,1280.0,5,1280.0,5,1257.0,4,1208.0,7,1214.0,10 +7,MarkovBlanket,Search,Backward search,1226.0,1224.0,9,1224.0,8,1249.0,5,1217.0,6,1224.0,8 +8,LOCO,Search,Leave-one-out,1206.0,1244.0,7,1130.0,12,1198.0,11,1181.0,9,1287.0,5 +9,SFS,Search,Forward search,1203.0,1240.0,8,1183.0,10,1209.0,8,1157.0,10,1236.0,7 +10,F-Test,Non-search,Correlation,1197.0,1191.0,11,1242.0,6,1199.0,10,1149.0,11,1214.0,9 +11,SBE,Search,Backward search,1190.0,1260.0,6,1082.0,13,1202.0,9,1200.0,8,1207.0,11 +12,GainRatio,Non-search,Information theory,1183.0,1179.0,12,1222.0,9,1190.0,12,1147.0,13,1186.0,12 +13,mRMR,Non-search,Information theory,1153.0,1151.0,13,1135.0,11,1182.0,13,1149.0,12,1155.0,13 +14,LaplacianScore,Non-search,Distance,1007.0,1035.0,14,1030.0,14,1027.0,14,941.0,15,999.0,15 +15,Random,Baseline,Baseline,1000.0,1000.0,15,1000.0,15,1000.0,15,1000.0,14,1000.0,14 diff --git a/data/subsets/tasks_regression/models_all/budgets_all/per_model_table.html b/data/subsets/tasks_regression/models_all/budgets_all/per_model_table.html new file mode 100644 index 0000000000000000000000000000000000000000..d933b18152b9f60da9a337e4fb2c803494d540df --- /dev/null +++ b/data/subsets/tasks_regression/models_all/budgets_all/per_model_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, all models, all budgets · per model
Elo per downstream model; medals mark the three best methods of each model. Click a column to sort.
#MethodAll models ↑LightGBM ↑LinearModel ↑RandomForest ↑RealMLP ↑TabICLv2 ↑
🥇1RFImportance1372🥇1411🥇1358🥇1372🥈1315🥇1422
🥈2CART1354🥈13871316🥈1368🥇1318🥈1396
🥉3Lasso1273🥉1298🥉133012191230🥉1304
4ElasticNet12701285🥈1336123812341275
5(R)ReliefF125012171225🥉1283🥉12401297
6MI124512801280125712081214
7MarkovBlanket122612241224124912171224
8LOCO120612441130119811811287
9SFS120312401183120911571236
10F-Test119711911242119911491214
11SBE119012601082120212001207
12GainRatio118311791222119011471186
13mRMR115311511135118211491155
14LaplacianScore1007103510301027941999
15Random100010001000100010001000
diff --git a/data/subsets/tasks_regression/models_lightgbm/budgets_0/pareto.html b/data/subsets/tasks_regression/models_lightgbm/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..f0986dc60f3bf715ab84fa9a29a04b579d87012c --- /dev/null +++ b/data/subsets/tasks_regression/models_lightgbm/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LightGBM, budget 1
Hover or tap a method for its values.
10-210-110010110210310402468101214Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMISBEElasticNetLassoF-TestSFSGainRatioMarkovBlanketmRMRLOCO(R)ReliefFRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_lightgbm/budgets_1/pareto.html b/data/subsets/tasks_regression/models_lightgbm/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..1a1c006730e81f1018f4bc9127e129c7d247aad7 --- /dev/null +++ b/data/subsets/tasks_regression/models_lightgbm/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LightGBM, budget 2
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTSFSMISBEElasticNetLassoGainRatioLOCO(R)ReliefFF-TestmRMRMarkovBlanketRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_lightgbm/budgets_2/pareto.html b/data/subsets/tasks_regression/models_lightgbm/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..0cddfc0643c0f154a85bc0ee359b8f6d253111ce --- /dev/null +++ b/data/subsets/tasks_regression/models_lightgbm/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LightGBM, budget 3
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTLOCOMISBELasso(R)ReliefFMarkovBlanketElasticNetSFSF-TestmRMRGainRatioRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_lightgbm/budgets_3/pareto.html b/data/subsets/tasks_regression/models_lightgbm/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..60e1a705433c0e3287f9eae99269b20780072067 --- /dev/null +++ b/data/subsets/tasks_regression/models_lightgbm/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LightGBM, budget 4
Hover or tap a method for its values.
10-210-1100101102103104012345678Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMILOCOF-Test(R)ReliefFMarkovBlanketLassoElasticNetSBEmRMRSFSGainRatioRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_lightgbm/budgets_4/pareto.html b/data/subsets/tasks_regression/models_lightgbm/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..47b6835f30c00bdaa94028e56f2aa362197e8312 --- /dev/null +++ b/data/subsets/tasks_regression/models_lightgbm/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LightGBM, budget 5
Hover or tap a method for its values.
10-210-11001011021031040123456Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalCARTRFImportanceMI(R)ReliefFSBEF-TestLOCOMarkovBlanketLassoElasticNetmRMRGainRatioSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_lightgbm/budgets_all/pareto.html b/data/subsets/tasks_regression/models_lightgbm/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..e664c14f939fe6edbebcb1142f6c3d689944d28f --- /dev/null +++ b/data/subsets/tasks_regression/models_lightgbm/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LightGBM, all budgets
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMISBELassoElasticNetLOCOF-TestSFSMarkovBlanket(R)ReliefFGainRatiomRMRRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_linearmodel/budgets_0/pareto.html b/data/subsets/tasks_regression/models_linearmodel/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..798d4b23c9e5ac93069544602232c12c5f605dbb --- /dev/null +++ b/data/subsets/tasks_regression/models_linearmodel/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LinearModel, budget 1
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTElasticNetMILassoF-TestMarkovBlanketSFSGainRatiomRMRLOCO(R)ReliefFSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_linearmodel/budgets_1/pareto.html b/data/subsets/tasks_regression/models_linearmodel/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..a9cdb825a0375ea4f334fe81076e96ad01eb0864 --- /dev/null +++ b/data/subsets/tasks_regression/models_linearmodel/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LinearModel, budget 2
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTElasticNetLassoMIF-TestGainRatioMarkovBlanketSFSmRMRLOCO(R)ReliefFSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_linearmodel/budgets_2/pareto.html b/data/subsets/tasks_regression/models_linearmodel/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..8a588795b011d88e929efc5de0875639a33f0b01 --- /dev/null +++ b/data/subsets/tasks_regression/models_linearmodel/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LinearModel, budget 3
Hover or tap a method for its values.
10-210-1100101102103104012345678Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMILassoElasticNetSFSMarkovBlanket(R)ReliefFF-TestGainRatioLOCOmRMRSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_linearmodel/budgets_3/pareto.html b/data/subsets/tasks_regression/models_linearmodel/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..c656f9abff2ea06ea7ba53797faaaea9fd8f3f3a --- /dev/null +++ b/data/subsets/tasks_regression/models_linearmodel/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LinearModel, budget 4
Hover or tap a method for its values.
10-210-110010110210310401234567Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMILassoElasticNetF-Test(R)ReliefFLOCOGainRatioMarkovBlanketSFSmRMRSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_linearmodel/budgets_4/pareto.html b/data/subsets/tasks_regression/models_linearmodel/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..0e14e0b56d0a8841a1695152dd7cdc3d791db6d0 --- /dev/null +++ b/data/subsets/tasks_regression/models_linearmodel/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LinearModel, budget 5
Hover or tap a method for its values.
10-210-110010110210310401234Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceMICART(R)ReliefFLassoElasticNetLOCOmRMRGainRatioF-TestMarkovBlanketSBESFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_linearmodel/budgets_all/pareto.html b/data/subsets/tasks_regression/models_linearmodel/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..e0f86b8d16dd84032e1b687b75a4d010e8619a48 --- /dev/null +++ b/data/subsets/tasks_regression/models_linearmodel/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, LinearModel, all budgets
Hover or tap a method for its values.
10-210-110010110210310402468Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIElasticNetLassoF-TestMarkovBlanketGainRatioSFS(R)ReliefFmRMRLOCOSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_randomforest/budgets_0/pareto.html b/data/subsets/tasks_regression/models_randomforest/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..982a350fc313826bcc9a198fe3edbc20c01d40c1 --- /dev/null +++ b/data/subsets/tasks_regression/models_randomforest/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RandomForest, budget 1
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalElasticNetRFImportanceLassoCARTF-TestMarkovBlanketSBEMI(R)ReliefFmRMRLOCOSFSGainRatioRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_randomforest/budgets_1/pareto.html b/data/subsets/tasks_regression/models_randomforest/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..2e3a87f05cf00e40bd7387b9a7219fdfe9b12a30 --- /dev/null +++ b/data/subsets/tasks_regression/models_randomforest/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RandomForest, budget 2
Hover or tap a method for its values.
10-210-110010110210310402.557.51012.51517.5Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIF-Test(R)ReliefFGainRatioElasticNetLassoSBEMarkovBlanketSFSLOCOmRMRRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_randomforest/budgets_2/pareto.html b/data/subsets/tasks_regression/models_randomforest/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..909584ece641fd7f59b64c11b169c811688b32a1 --- /dev/null +++ b/data/subsets/tasks_regression/models_randomforest/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RandomForest, budget 3
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalCARTRFImportanceMIF-TestLOCO(R)ReliefFElasticNetLassoMarkovBlanketSBEmRMRGainRatioSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_randomforest/budgets_3/pareto.html b/data/subsets/tasks_regression/models_randomforest/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..336c662826084bde0cc86e8e25d71a0b4e55ee5a --- /dev/null +++ b/data/subsets/tasks_regression/models_randomforest/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RandomForest, budget 4
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalCARTRFImportanceMI(R)ReliefFF-TestLOCOLassoElasticNetMarkovBlanketmRMRGainRatioSBESFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_randomforest/budgets_4/pareto.html b/data/subsets/tasks_regression/models_randomforest/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..fb210376ef00c49190c87858d8e3dcdcd4c824fc --- /dev/null +++ b/data/subsets/tasks_regression/models_randomforest/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RandomForest, budget 5
Hover or tap a method for its values.
10-210-110010110210310400.511.522.533.54Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportance(R)ReliefFCARTMIElasticNetMarkovBlanketLassoLOCOF-TestmRMRSFSSBEGainRatioRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_randomforest/budgets_all/pareto.html b/data/subsets/tasks_regression/models_randomforest/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..cb231b4fcd33fcb00e8b7e45bb5829e1e79e3102 --- /dev/null +++ b/data/subsets/tasks_regression/models_randomforest/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RandomForest, all budgets
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIElasticNetLassoF-TestMarkovBlanket(R)ReliefFSBELOCOGainRatiomRMRSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_0/elo_explorer.html b/data/subsets/tasks_regression/models_realmlp/budgets_0/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..1792873f1a0a29fac939e08bfaeea9e85d8cc6b7 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_0/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, RealMLP, budget 1
Hover or tap a method for its values.
8001000120014001600EloLaplacianScoreRandomLOCOGainRatio(R)ReliefFSBEmRMRF-TestSFSMILassoMarkovBlanketCARTElasticNetRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_0/leaderboard_table.html b/data/subsets/tasks_regression/models_realmlp/budgets_0/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..3045b30ff58f129a4c8441b183e3c5d46d35f304 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_0/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, RealMLP, budget 1
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1327+187 / −1435.635.60.7330.8501.00.0
🥈2ElasticNetRegularizationnon-search1318+115 / −806.159.00.5230.9380.20.0
🥉3CARTTreenon-search1316+158 / −1255.626.50.6330.8250.10.0
4MarkovBlanketBackward searchsearch1296+167 / −1216.875.80.4290.7500.10.0
5LassoRegularizationnon-search1264+133 / −996.529.20.5410.9220.10.0
6MIInformation theorynon-search1262+167 / −1336.686.10.7581.0000.50.0
7SFSForward searchsearch1223+167 / −1767.0311.40.3630.792244.926.7
8F-TestCorrelationnon-search1219+136 / −1077.279.30.8180.9690.00.0
9mRMRInformation theorynon-search1202+165 / −1628.7710.40.5170.6690.40.0
10(R)ReliefFDistancenon-search1174+91 / −1038.6813.00.6290.953293.30.0
11SBEBackward searchsearch1174+171 / −1768.738.80.2040.611805.940.0
12GainRatioInformation theorynon-search1096+136 / −17210.0011.70.6290.8120.40.0
13LOCOLeave-one-outsearch1073+166 / −2199.5214.60.3120.685123.90.0
14RandomBaseline1000+107 / −19710.5716.40.0000.4740.00.0
15LaplacianScoreDistancenon-search928+141 / −24511.9718.70.6990.8250.10.0
diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_0/pareto.csv b/data/subsets/tasks_regression/models_realmlp/budgets_0/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..2a9c6e87483b63ef85859401ff2874990384878d --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_0/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,13.0087,236.0143,10,True +CART,12.0413,22.152,10,True +MarkovBlanket,11.972,42.2852,10,False +MI,10.3869,130.657,10,False +F-Test,9.4036,1.1072,10,True +ElasticNet,8.9902,37.476,10,False +Lasso,8.7359,13.508,10,False +SBE,8.5661,27589.2843,10,False +mRMR,8.3725,94.1047,10,False +SFS,6.5102,14012.5242,10,False +GainRatio,6.0894,89.9994,10,False +(R)ReliefF,4.5656,11522.9584,10,False +LOCO,3.6777,8366.1616,10,False +Random,0.0,0.0386,10,True +LaplacianScore,-3.1296,4.6228,10,False diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_0/pareto.html b/data/subsets/tasks_regression/models_realmlp/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..56124701a431974cdf56a4f3ed47f4bdc2727094 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RealMLP, budget 1
Hover or tap a method for its values.
10-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMarkovBlanketMIF-TestElasticNetLassoSBEmRMRSFSGainRatio(R)ReliefFLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_0/website_leaderboard.csv b/data/subsets/tasks_regression/models_realmlp/budgets_0/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..b95f01bc136565ee2b68d7fcf8c488fca356c82b --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_0/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1327.0,1184.0,1514.0,5.63,66.9,5.6,,0.733,0.85,1.0,0.0,10,30 +2,ElasticNet,Non-search,Regularization,1318.0,1238.0,1433.0,6.15,63.2,9.0,,0.523,0.938,0.2,0.0,10,30 +3,CART,Non-search,Tree,1316.0,1191.0,1474.0,5.62,67.0,6.5,,0.633,0.825,0.1,0.0,10,30 +4,MarkovBlanket,Search,Backward search,1296.0,1175.0,1463.0,6.87,58.1,5.8,,0.429,0.75,0.1,0.0,10,30 +5,Lasso,Non-search,Regularization,1264.0,1165.0,1397.0,6.52,60.6,9.2,,0.541,0.922,0.1,0.0,10,30 +6,MI,Non-search,Information theory,1262.0,1129.0,1429.0,6.68,59.4,6.1,,0.758,1.0,0.5,0.0,10,30 +7,SFS,Search,Forward search,1223.0,1047.0,1390.0,7.03,56.9,11.4,,0.363,0.792,244.9,26.7,10,30 +8,F-Test,Non-search,Correlation,1219.0,1112.0,1355.0,7.27,55.2,9.3,,0.818,0.969,0.0,0.0,10,30 +9,mRMR,Non-search,Information theory,1202.0,1040.0,1367.0,8.77,44.5,10.4,,0.517,0.669,0.4,0.0,10,30 +10,(R)ReliefF,Non-search,Distance,1174.0,1071.0,1265.0,8.68,45.1,13.0,,0.629,0.953,293.3,0.0,10,30 +11,SBE,Search,Backward search,1174.0,998.0,1345.0,8.73,44.8,8.8,,0.204,0.611,805.9,40.0,10,30 +12,GainRatio,Non-search,Information theory,1096.0,924.0,1232.0,10.0,35.7,11.7,,0.629,0.812,0.4,0.0,10,30 +13,LOCO,Search,Leave-one-out,1073.0,854.0,1239.0,9.52,39.2,14.6,,0.312,0.685,123.9,0.0,10,30 +14,Random,Baseline,Baseline,1000.0,803.0,1107.0,10.57,31.7,16.4,,0.0,0.474,0.0,0.0,10,30 +15,LaplacianScore,Non-search,Distance,928.0,683.0,1069.0,11.97,21.7,18.7,,0.699,0.825,0.1,0.0,10,30 diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_1/elo_explorer.html b/data/subsets/tasks_regression/models_realmlp/budgets_1/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..286a728bb80e5473882a2ad2a6f51d09cc72bb03 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_1/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, RealMLP, budget 2
Hover or tap a method for its values.
8001000120014001600EloLaplacianScoreRandommRMRLOCO(R)ReliefFF-TestMIGainRatioSFSLassoElasticNetSBEMarkovBlanketCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_1/leaderboard_table.html b/data/subsets/tasks_regression/models_realmlp/budgets_1/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..9143d2be02495c6a457a413797399eb1a4669806 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_1/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, RealMLP, budget 2
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1369+184 / −1285.254.00.7190.8021.00.0
🥈2CARTTreenon-search1345+167 / −1264.684.20.5610.7630.10.0
🥉3MarkovBlanketBackward searchsearch1304+142 / −1107.028.00.3390.7280.10.0
4SBEBackward searchsearch1299+145 / −1337.088.10.1950.736758.940.0
5ElasticNetRegularizationnon-search1278+138 / −1256.888.40.6280.9000.20.0
6LassoRegularizationnon-search1262+142 / −1216.778.40.6330.8380.10.0
7SFSForward searchsearch1226+144 / −1367.538.10.2620.711423.640.0
8GainRatioInformation theorynon-search1209+146 / −1368.578.00.6320.8050.40.0
9MIInformation theorynon-search1201+150 / −1317.776.60.8180.9690.60.0
10F-TestCorrelationnon-search1186+129 / −1328.689.50.7900.9380.00.0
11(R)ReliefFDistancenon-search1160+124 / −1178.4310.60.6130.916292.20.0
12LOCOLeave-one-outsearch1134+163 / −1819.0310.50.2320.687124.00.0
13mRMRInformation theorynon-search1120+119 / −1408.659.90.5350.6540.50.0
14RandomBaseline1000+155 / −25311.6517.80.0060.5150.00.0
15LaplacianScoreDistancenon-search965+122 / −21012.0015.90.7200.8430.10.0
diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_1/pareto.csv b/data/subsets/tasks_regression/models_realmlp/budgets_1/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..03cbfd266b552a79968b10f5a0eb045a0d6cff0a --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_1/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,15.2404,237.3097,10,True +CART,15.0307,22.3118,10,True +MI,12.508,130.7572,10,False +MarkovBlanket,11.7601,41.6333,10,False +Lasso,11.5544,12.8839,10,True +GainRatio,11.5509,92.1542,10,False +ElasticNet,11.5211,36.7976,10,False +SFS,11.1022,18858.4011,10,False +SBE,11.0501,26168.4777,10,False +F-Test,10.2516,1.0007,10,True +mRMR,9.0525,104.1081,10,False +(R)ReliefF,8.9452,11535.7035,10,False +LOCO,8.873,8128.311,10,False +LaplacianScore,2.8879,4.7412,10,False +Random,0.0,0.0034,10,True diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_1/pareto.html b/data/subsets/tasks_regression/models_realmlp/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..7eb0e33be4ef390bc3399c9fce483647da741792 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RealMLP, budget 2
Hover or tap a method for its values.
10-210-11001011021031040246810121416Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIMarkovBlanketLassoGainRatioElasticNetSFSSBEF-TestmRMR(R)ReliefFLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_1/website_leaderboard.csv b/data/subsets/tasks_regression/models_realmlp/budgets_1/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..d233e9f6c912a2b70bef72e216acb326be05c31f --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_1/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1369.0,1241.0,1553.0,5.25,69.6,4.0,,0.719,0.802,1.0,0.0,10,30 +2,CART,Non-search,Tree,1345.0,1219.0,1512.0,4.68,73.7,4.2,,0.561,0.763,0.1,0.0,10,30 +3,MarkovBlanket,Search,Backward search,1304.0,1194.0,1446.0,7.02,57.0,8.0,,0.339,0.728,0.1,0.0,10,30 +4,SBE,Search,Backward search,1299.0,1166.0,1444.0,7.08,56.6,8.1,,0.195,0.736,758.9,40.0,10,30 +5,ElasticNet,Non-search,Regularization,1278.0,1153.0,1416.0,6.88,58.0,8.4,,0.628,0.9,0.2,0.0,10,30 +6,Lasso,Non-search,Regularization,1262.0,1141.0,1404.0,6.77,58.8,8.4,,0.633,0.838,0.1,0.0,10,30 +7,SFS,Search,Forward search,1226.0,1090.0,1370.0,7.53,53.3,8.1,,0.262,0.711,423.6,40.0,10,30 +8,GainRatio,Non-search,Information theory,1209.0,1073.0,1355.0,8.57,46.0,8.0,,0.632,0.805,0.4,0.0,10,30 +9,MI,Non-search,Information theory,1201.0,1070.0,1351.0,7.77,51.7,6.6,,0.818,0.969,0.6,0.0,10,30 +10,F-Test,Non-search,Correlation,1186.0,1054.0,1315.0,8.68,45.1,9.5,,0.79,0.938,0.0,0.0,10,30 +11,(R)ReliefF,Non-search,Distance,1160.0,1043.0,1284.0,8.43,46.9,10.6,,0.613,0.916,292.2,0.0,10,30 +12,LOCO,Search,Leave-one-out,1134.0,953.0,1297.0,9.03,42.6,10.5,,0.232,0.687,124.0,0.0,10,30 +13,mRMR,Non-search,Information theory,1120.0,980.0,1239.0,8.65,45.4,9.9,,0.535,0.654,0.5,0.0,10,30 +14,Random,Baseline,Baseline,1000.0,747.0,1155.0,11.65,23.9,17.8,,0.006,0.515,0.0,0.0,10,30 +15,LaplacianScore,Non-search,Distance,965.0,755.0,1087.0,12.0,21.4,15.9,,0.72,0.843,0.1,0.0,10,30 diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_2/elo_explorer.html b/data/subsets/tasks_regression/models_realmlp/budgets_2/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..0458db07813fc3159b210455c43bf95ebfc6f86d --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_2/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, RealMLP, budget 3
Hover or tap a method for its values.
80010001200140016001800EloLaplacianScoreRandomF-TestmRMRSFSGainRatioMILassoElasticNetLOCOSBEMarkovBlanket(R)ReliefFRFImportanceCARTstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_2/leaderboard_table.html b/data/subsets/tasks_regression/models_realmlp/budgets_2/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..229dbb5d6608d577e407370626579cc5056c27d3 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_2/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, RealMLP, budget 3
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1490+269 / −1544.022.50.6780.7051.00.0
🥈2CARTTreenon-search1490+225 / −1354.623.90.5170.6670.10.0
🥉3(R)ReliefFDistancenon-search1376+163 / −1116.405.20.7040.842280.20.0
4MarkovBlanketBackward searchsearch1326+147 / −1428.179.80.3850.6820.10.0
5SBEBackward searchsearch1299+193 / −1588.009.70.1510.608633.840.0
6LOCOLeave-one-outsearch1283+107 / −1057.737.70.1600.668129.30.0
7LassoRegularizationnon-search1280+155 / −1817.689.40.4910.7840.10.0
8ElasticNetRegularizationnon-search1280+152 / −1767.979.30.5600.8360.20.0
9MIInformation theorynon-search1261+141 / −1277.737.10.9170.9110.50.0
10GainRatioInformation theorynon-search1243+154 / −1368.0810.10.5640.7410.40.0
11SFSForward searchsearch1229+123 / −1278.309.30.2600.645574.540.0
12mRMRInformation theorynon-search1199+122 / −1608.6510.30.5420.5930.50.0
13F-TestCorrelationnon-search1110+143 / −2109.6510.20.7640.8730.00.0
14RandomBaseline1000+118 / −21410.7014.9-0.0030.4900.00.0
15LaplacianScoreDistancenon-search912+107 / −21712.3016.30.7650.8490.20.0
diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_2/pareto.csv b/data/subsets/tasks_regression/models_realmlp/budgets_2/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..1aba947581390bf691dd97cd91dc2dd09c8c4c77 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_2/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,14.1282,239.4535,10,True +CART,12.8977,22.2967,10,True +(R)ReliefF,11.7135,11467.5529,10,False +MI,9.8342,131.7466,10,False +LOCO,9.0928,8393.8255,10,False +Lasso,7.7054,12.8714,10,True +MarkovBlanket,7.219,41.4653,10,False +ElasticNet,7.0037,36.9386,10,False +SFS,6.8774,21268.7788,10,False +SBE,6.7157,23459.2444,10,False +GainRatio,6.5586,89.8326,10,False +mRMR,6.3846,118.3326,10,False +F-Test,6.11,0.989,10,True +Random,0.0,0.0046,10,True +LaplacianScore,-1.7734,5.0184,10,False diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_2/pareto.html b/data/subsets/tasks_regression/models_realmlp/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..5596573a06d665c67eb2126959bb17b610bebd62 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RealMLP, budget 3
Hover or tap a method for its values.
10-210-110010110210310402468101214Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCART(R)ReliefFMILOCOLassoMarkovBlanketElasticNetSFSSBEGainRatiomRMRF-TestRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_2/website_leaderboard.csv b/data/subsets/tasks_regression/models_realmlp/budgets_2/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..5364f16072a1438c59522214f10418c3b8b814c2 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_2/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1490.0,1336.0,1759.0,4.02,78.4,2.5,,0.678,0.705,1.0,0.0,10,30 +2,CART,Non-search,Tree,1490.0,1355.0,1715.0,4.62,74.2,3.9,,0.517,0.667,0.1,0.0,10,30 +3,(R)ReliefF,Non-search,Distance,1376.0,1265.0,1539.0,6.4,61.4,5.2,,0.704,0.842,280.2,0.0,10,30 +4,MarkovBlanket,Search,Backward search,1326.0,1184.0,1473.0,8.17,48.8,9.8,,0.385,0.682,0.1,0.0,10,30 +5,SBE,Search,Backward search,1299.0,1141.0,1492.0,8.0,50.0,9.7,,0.151,0.608,633.8,40.0,10,30 +6,LOCO,Search,Leave-one-out,1283.0,1178.0,1390.0,7.73,51.9,7.7,,0.16,0.668,129.3,0.0,10,30 +7,Lasso,Non-search,Regularization,1280.0,1099.0,1435.0,7.68,52.3,9.4,,0.491,0.784,0.1,0.0,10,30 +8,ElasticNet,Non-search,Regularization,1280.0,1104.0,1432.0,7.97,50.2,9.3,,0.56,0.836,0.2,0.0,10,30 +9,MI,Non-search,Information theory,1261.0,1134.0,1402.0,7.73,51.9,7.1,,0.917,0.911,0.5,0.0,10,30 +10,GainRatio,Non-search,Information theory,1243.0,1107.0,1397.0,8.08,49.4,10.1,,0.564,0.741,0.4,0.0,10,30 +11,SFS,Search,Forward search,1229.0,1102.0,1352.0,8.3,47.9,9.3,,0.26,0.645,574.5,40.0,10,30 +12,mRMR,Non-search,Information theory,1199.0,1039.0,1321.0,8.65,45.4,10.3,,0.542,0.593,0.5,0.0,10,30 +13,F-Test,Non-search,Correlation,1110.0,900.0,1253.0,9.65,38.2,10.2,,0.764,0.873,0.0,0.0,10,30 +14,Random,Baseline,Baseline,1000.0,786.0,1118.0,10.7,30.7,14.9,,-0.003,0.49,0.0,0.0,10,30 +15,LaplacianScore,Non-search,Distance,912.0,695.0,1019.0,12.3,19.3,16.3,,0.765,0.849,0.2,0.0,10,30 diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_3/elo_explorer.html b/data/subsets/tasks_regression/models_realmlp/budgets_3/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..d7bf3be7ca8f9b7c5fe653e9e7fd684d9317879f --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_3/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, RealMLP, budget 4
Hover or tap a method for its values.
6008001000120014001600EloLaplacianScoreRandomMarkovBlanketGainRatioMISBEF-TestSFSElasticNetmRMRLassoLOCORFImportanceCART(R)ReliefFstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_3/leaderboard_table.html b/data/subsets/tasks_regression/models_realmlp/budgets_3/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..c1a169a0570f8a12c8510ee04fd4aa09122add29 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_3/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, RealMLP, budget 4
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1(R)ReliefFDistancenon-search1334+174 / −1285.602.60.7310.741283.70.0
🥈2CARTTreenon-search1329+216 / −1545.534.80.4890.5460.10.0
🥉3RFImportanceTreenon-search1289+206 / −1455.733.90.7340.6071.00.0
4LOCOLeave-one-outsearch1273+120 / −926.575.80.0920.621124.610.0
5LassoRegularizationnon-search1214+172 / −1587.438.10.4230.7090.10.0
6mRMRInformation theorynon-search1208+108 / −1217.227.50.5540.5330.50.0
7ElasticNetRegularizationnon-search1176+123 / −1327.907.60.4780.7250.20.0
8F-TestCorrelationnon-search1175+134 / −1267.675.50.7130.7780.00.0
9SFSForward searchsearch1175+143 / −1107.776.60.2180.598683.540.0
10SBEBackward searchsearch1170+152 / −1378.038.10.0890.588431.140.0
11MIInformation theorynon-search1129+118 / −1238.476.00.7960.7740.60.0
12GainRatioInformation theorynon-search1115+153 / −1639.127.20.5590.6350.40.0
13MarkovBlanketBackward searchsearch1113+155 / −1699.578.60.4150.6340.10.0
14RandomBaseline1000+157 / −22910.5712.30.0030.5200.00.0
15LaplacianScoreDistancenon-search876+164 / −37812.8316.00.7590.7270.10.0
diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_3/pareto.csv b/data/subsets/tasks_regression/models_realmlp/budgets_3/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..7e3e1141db3fe50100f3ac8002317addde51fe44 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_3/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +(R)ReliefF,10.6564,11536.5857,10,True +RFImportance,8.8053,239.096,10,True +F-Test,8.0275,0.9839,10,True +CART,7.8234,22.3953,10,False +MI,7.1719,131.0627,10,False +LOCO,6.9604,8175.5948,10,False +SFS,6.7957,25420.1496,10,False +GainRatio,6.3657,90.5082,10,False +mRMR,5.6951,152.5391,10,False +Lasso,5.2387,13.0283,10,False +ElasticNet,5.0952,36.4421,10,False +MarkovBlanket,4.712,41.3868,10,False +SBE,4.3083,19547.0957,10,False +Random,0.0,0.0034,10,True +LaplacianScore,-6.1192,4.9563,10,False diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_3/pareto.html b/data/subsets/tasks_regression/models_realmlp/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..4fb685ef5a65ef562597357648825955abc12d24 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RealMLP, budget 4
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimal(R)ReliefFRFImportanceF-TestCARTMILOCOSFSGainRatiomRMRLassoElasticNetMarkovBlanketSBERandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_3/website_leaderboard.csv b/data/subsets/tasks_regression/models_realmlp/budgets_3/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..133bd7346df10103c3fd46b239ce44675b24b7a8 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_3/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,(R)ReliefF,Non-search,Distance,1334.0,1206.0,1508.0,5.6,67.1,2.6,,0.731,0.741,283.7,0.0,10,30 +2,CART,Non-search,Tree,1329.0,1175.0,1545.0,5.53,67.6,4.8,,0.489,0.546,0.1,0.0,10,30 +3,RFImportance,Non-search,Tree,1289.0,1144.0,1495.0,5.73,66.2,3.9,,0.734,0.607,1.0,0.0,10,30 +4,LOCO,Search,Leave-one-out,1273.0,1181.0,1393.0,6.57,60.2,5.8,,0.092,0.621,124.6,10.0,10,30 +5,Lasso,Non-search,Regularization,1214.0,1056.0,1386.0,7.43,54.0,8.1,,0.423,0.709,0.1,0.0,10,30 +6,mRMR,Non-search,Information theory,1208.0,1087.0,1316.0,7.22,55.6,7.5,,0.554,0.533,0.5,0.0,10,30 +7,ElasticNet,Non-search,Regularization,1176.0,1044.0,1299.0,7.9,50.7,7.6,,0.478,0.725,0.2,0.0,10,30 +8,F-Test,Non-search,Correlation,1175.0,1049.0,1309.0,7.67,52.4,5.5,,0.713,0.778,0.0,0.0,10,30 +9,SFS,Search,Forward search,1175.0,1065.0,1318.0,7.77,51.7,6.6,,0.218,0.598,683.5,40.0,10,30 +10,SBE,Search,Backward search,1170.0,1033.0,1322.0,8.03,49.8,8.1,,0.089,0.588,431.1,40.0,10,30 +11,MI,Non-search,Information theory,1129.0,1006.0,1247.0,8.47,46.7,6.0,,0.796,0.774,0.6,0.0,10,30 +12,GainRatio,Non-search,Information theory,1115.0,952.0,1268.0,9.12,42.0,7.2,,0.559,0.635,0.4,0.0,10,30 +13,MarkovBlanket,Search,Backward search,1113.0,944.0,1268.0,9.57,38.8,8.6,,0.415,0.634,0.1,0.0,10,30 +14,Random,Baseline,Baseline,1000.0,771.0,1157.0,10.57,31.7,12.3,,0.003,0.52,0.0,0.0,10,30 +15,LaplacianScore,Non-search,Distance,876.0,498.0,1040.0,12.83,15.5,16.0,,0.759,0.727,0.1,0.0,10,30 diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_4/elo_explorer.html b/data/subsets/tasks_regression/models_realmlp/budgets_4/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..b44e4163c5cf98ee668783b35c430f61148fbeca --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_4/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, RealMLP, budget 5
Hover or tap a method for its values.
800900100011001200130014001500EloSFSLaplacianScoreRandommRMRF-TestMarkovBlanketGainRatioSBEElasticNetRFImportanceLassoLOCOCART(R)ReliefFMIstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_4/leaderboard_table.html b/data/subsets/tasks_regression/models_realmlp/budgets_4/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..4bb32e7586d5101025b3903622f666aa66643434 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_4/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, RealMLP, budget 5
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1MIInformation theorynon-search1232+168 / −1176.171.50.7590.6480.60.0
🥈2(R)ReliefFDistancenon-search1216+181 / −1356.534.90.6650.634279.70.0
🥉3CARTTreenon-search1185+156 / −1327.033.60.5060.4630.10.0
4LOCOLeave-one-outsearch1180+125 / −1117.104.60.0520.557124.220.0
5LassoRegularizationnon-search1178+183 / −1647.085.50.4150.6030.10.0
6RFImportanceTreenon-search1175+134 / −1356.373.30.7060.4811.00.0
7ElasticNetRegularizationnon-search1167+205 / −1887.154.60.4570.6170.20.0
8SBEBackward searchsearch1106+131 / −1498.705.40.0610.541224.540.0
9GainRatioInformation theorynon-search1104+114 / −1118.384.60.5040.5530.40.0
10MarkovBlanketBackward searchsearch1101+119 / −1508.274.30.4350.5610.10.0
11F-TestCorrelationnon-search1082+136 / −1368.034.60.5510.6880.00.0
12mRMRInformation theorynon-search1049+115 / −1408.326.10.5750.5170.50.0
13RandomBaseline1000+79 / −12010.338.0-0.0010.4930.00.0
14LaplacianScoreDistancenon-search989+161 / −19910.2310.40.7550.6340.20.0
15SFSForward searchsearch967+95 / −12910.307.10.1200.533784.240.0
diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_4/pareto.csv b/data/subsets/tasks_regression/models_realmlp/budgets_4/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..53d30e3e21022338948adb9e485128d2d10d8fb9 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_4/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +MI,7.2156,132.5607,10,True +CART,5.0711,22.3383,10,True +RFImportance,4.8254,236.3657,10,False +MarkovBlanket,4.6288,39.1486,10,False +GainRatio,4.1235,90.5059,10,False +F-Test,4.0864,0.9777,10,True +LOCO,4.0676,8052.3887,10,False +ElasticNet,3.9776,36.1017,10,False +SBE,3.0391,14489.685,10,False +Lasso,2.6342,12.699,10,False +(R)ReliefF,2.0344,11495.6437,10,False +mRMR,1.8153,236.3546,10,False +SFS,0.3332,27652.0766,10,False +Random,0.0,0.0095,10,True +LaplacianScore,-4.06,4.8512,10,False diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_4/pareto.html b/data/subsets/tasks_regression/models_realmlp/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..7bd0eeb5a379adedc94981c27cebd69d4234ed85 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RealMLP, budget 5
Hover or tap a method for its values.
10-210-110010110210310401234567Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalMICARTRFImportanceMarkovBlanketGainRatioF-TestLOCOElasticNetSBELasso(R)ReliefFmRMRSFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_4/website_leaderboard.csv b/data/subsets/tasks_regression/models_realmlp/budgets_4/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..e0b3159e2f27ae1d71337ac1b214a9e7d719213d --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_4/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,MI,Non-search,Information theory,1232.0,1115.0,1400.0,6.17,63.1,1.5,,0.759,0.648,0.6,0.0,10,30 +2,(R)ReliefF,Non-search,Distance,1216.0,1081.0,1397.0,6.53,60.5,4.9,,0.665,0.634,279.7,0.0,10,30 +3,CART,Non-search,Tree,1185.0,1053.0,1341.0,7.03,56.9,3.6,,0.506,0.463,0.1,0.0,10,30 +4,LOCO,Search,Leave-one-out,1180.0,1069.0,1305.0,7.1,56.4,4.6,,0.052,0.557,124.2,20.0,10,30 +5,Lasso,Non-search,Regularization,1178.0,1014.0,1361.0,7.08,56.6,5.5,,0.415,0.603,0.1,0.0,10,30 +6,RFImportance,Non-search,Tree,1175.0,1040.0,1309.0,6.37,61.7,3.3,,0.706,0.481,1.0,0.0,10,30 +7,ElasticNet,Non-search,Regularization,1167.0,979.0,1372.0,7.15,56.1,4.6,,0.457,0.617,0.2,0.0,10,30 +8,SBE,Search,Backward search,1106.0,957.0,1237.0,8.7,45.0,5.4,,0.061,0.541,224.5,40.0,10,30 +9,GainRatio,Non-search,Information theory,1104.0,993.0,1218.0,8.38,47.3,4.6,,0.504,0.553,0.4,0.0,10,30 +10,MarkovBlanket,Search,Backward search,1101.0,951.0,1220.0,8.27,48.1,4.3,,0.435,0.561,0.1,0.0,10,30 +11,F-Test,Non-search,Correlation,1082.0,946.0,1218.0,8.03,49.8,4.6,,0.551,0.688,0.0,0.0,10,30 +12,mRMR,Non-search,Information theory,1049.0,909.0,1164.0,8.32,47.7,6.1,,0.575,0.517,0.5,0.0,10,30 +13,Random,Baseline,Baseline,1000.0,880.0,1079.0,10.33,33.3,8.0,,-0.001,0.493,0.0,0.0,10,30 +14,LaplacianScore,Non-search,Distance,989.0,790.0,1150.0,10.23,34.1,10.4,,0.755,0.634,0.2,0.0,10,30 +15,SFS,Search,Forward search,967.0,838.0,1062.0,10.3,33.6,7.1,,0.12,0.533,784.2,40.0,10,30 diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_all/elo_explorer.html b/data/subsets/tasks_regression/models_realmlp/budgets_all/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..14a974a8fca67b6253becd3aaeebb6842aa6302b --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_all/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, RealMLP, all budgets
Hover or tap a method for its values.
800900100011001200130014001500EloLaplacianScoreRandomGainRatioF-TestmRMRSFSLOCOSBEMIMarkovBlanketLassoElasticNet(R)ReliefFRFImportanceCARTstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_all/leaderboard_table.html b/data/subsets/tasks_regression/models_realmlp/budgets_all/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..0e81240a4168537606e9939ff12b0edcd885e820 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_all/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, RealMLP, all budgets
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1CARTTreenon-search1318+138 / −1105.504.60.5410.6530.10.0
🥈2RFImportanceTreenon-search1315+125 / −1135.403.90.7140.6891.00.0
🥉3(R)ReliefFDistancenon-search1240+88 / −677.137.30.6690.817291.20.0
4ElasticNetRegularizationnon-search1234+98 / −957.217.80.5290.8030.20.0
5LassoRegularizationnon-search1230+108 / −1077.108.10.5000.7710.10.0
6MarkovBlanketBackward searchsearch1217+87 / −867.987.30.4010.6710.10.0
7MIInformation theorynon-search1208+109 / −957.365.40.8100.8600.60.0
8SBEBackward searchsearch1200+108 / −1168.118.00.1400.617624.840.0
9LOCOLeave-one-outsearch1181+81 / −767.998.60.1700.643124.56.0
10SFSForward searchsearch1157+60 / −598.198.50.2550.656502.237.3
11F-TestCorrelationnon-search1149+101 / −908.267.80.7270.8490.00.0
12mRMRInformation theorynon-search1149+82 / −978.328.80.5450.5930.50.0
13GainRatioInformation theorynon-search1147+87 / −1028.838.30.5780.7090.40.0
14RandomBaseline1000+86 / −12010.7613.90.0010.4980.00.0
15LaplacianScoreDistancenon-search941+122 / −17811.8715.50.7400.7760.10.0
diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_all/pareto.csv b/data/subsets/tasks_regression/models_realmlp/budgets_all/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..6229f53beb0672bbb6cb286bdef2de975eb9cf4c --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_all/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,12.032,237.6478,10,True +CART,11.3754,22.2988,10,True +MI,10.2987,131.3568,10,False +MarkovBlanket,9.0316,41.1838,10,False +ElasticNet,8.0419,36.7512,10,False +F-Test,8.012,1.0117,10,True +Lasso,7.9758,12.9981,10,False +(R)ReliefF,7.8226,11511.6888,10,False +GainRatio,7.5962,90.6001,10,False +SBE,7.5471,22250.7574,10,False +SFS,7.066,21442.3861,10,False +mRMR,6.7912,141.0878,10,False +LOCO,6.7843,8223.2563,10,False +Random,0.0,0.0119,10,True +LaplacianScore,-1.4706,4.838,10,False diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_all/pareto.html b/data/subsets/tasks_regression/models_realmlp/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..d8a4ea94a5f3f813d3e3f00db81d628b596c5273 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, RealMLP, all budgets
Hover or tap a method for its values.
10-210-1100101102103104024681012Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMIMarkovBlanketElasticNetF-TestLasso(R)ReliefFGainRatioSBESFSmRMRLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_realmlp/budgets_all/website_leaderboard.csv b/data/subsets/tasks_regression/models_realmlp/budgets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..dfd9e84f61e9ac43d99a57a5b2eb3ae4ef271b86 --- /dev/null +++ b/data/subsets/tasks_regression/models_realmlp/budgets_all/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,CART,Non-search,Tree,1318.0,1208.0,1456.0,5.5,67.9,4.6,,0.541,0.653,0.1,0.0,10,150 +2,RFImportance,Non-search,Tree,1315.0,1202.0,1440.0,5.4,68.6,3.9,,0.714,0.689,1.0,0.0,10,150 +3,(R)ReliefF,Non-search,Distance,1240.0,1173.0,1328.0,7.13,56.2,7.3,,0.669,0.817,291.2,0.0,10,150 +4,ElasticNet,Non-search,Regularization,1234.0,1139.0,1332.0,7.21,55.6,7.8,,0.529,0.803,0.2,0.0,10,150 +5,Lasso,Non-search,Regularization,1230.0,1123.0,1338.0,7.1,56.4,8.1,,0.5,0.771,0.1,0.0,10,150 +6,MarkovBlanket,Search,Backward search,1217.0,1131.0,1304.0,7.98,50.2,7.3,,0.401,0.671,0.1,0.0,10,150 +7,MI,Non-search,Information theory,1208.0,1113.0,1317.0,7.36,54.6,5.4,,0.81,0.86,0.6,0.0,10,150 +8,SBE,Search,Backward search,1200.0,1084.0,1308.0,8.11,49.2,8.0,,0.14,0.617,624.8,40.0,10,150 +9,LOCO,Search,Leave-one-out,1181.0,1105.0,1262.0,7.99,50.1,8.6,,0.17,0.643,124.5,6.0,10,150 +10,SFS,Search,Forward search,1157.0,1098.0,1217.0,8.19,48.7,8.5,,0.255,0.656,502.2,37.3,10,150 +11,F-Test,Non-search,Correlation,1149.0,1059.0,1250.0,8.26,48.1,7.8,,0.727,0.849,0.0,0.0,10,150 +12,mRMR,Non-search,Information theory,1149.0,1052.0,1231.0,8.32,47.7,8.8,,0.545,0.593,0.5,0.0,10,150 +13,GainRatio,Non-search,Information theory,1147.0,1045.0,1234.0,8.83,44.1,8.3,,0.578,0.709,0.4,0.0,10,150 +14,Random,Baseline,Baseline,1000.0,880.0,1086.0,10.76,30.3,13.9,,0.001,0.498,0.0,0.0,10,150 +15,LaplacianScore,Non-search,Distance,941.0,763.0,1063.0,11.87,22.4,15.5,,0.74,0.776,0.1,0.0,10,150 diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_0/elo_explorer.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_0/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..906fdd9b5f2b2705e692ef84d37dc20b55d2835e --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_0/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 1
Hover or tap a method for its values.
8001000120014001600EloLaplacianScoreRandommRMRGainRatioMI(R)ReliefFSBEF-TestLOCOMarkovBlanketLassoElasticNetSFSCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_0/leaderboard_table.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_0/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..80136829029f0e01881c7367ffbd1416b628c74f --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_0/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 1
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1361+188 / −1324.884.20.7330.8501.00.0
🥈2CARTTreenon-search1324+122 / −895.355.30.6330.8250.10.0
🥉3SFSForward searchsearch1288+213 / −1596.357.70.3630.792245.630.0
4ElasticNetRegularizationnon-search1261+101 / −717.256.30.5230.9380.20.0
5MarkovBlanketBackward searchsearch1234+200 / −1667.226.10.4290.7500.20.0
6LassoRegularizationnon-search1234+92 / −647.376.40.5410.9220.10.0
7LOCOLeave-one-outsearch1178+146 / −1627.9010.10.3120.685129.30.0
8F-TestCorrelationnon-search1163+149 / −1397.737.00.8180.9690.00.0
9(R)ReliefFDistancenon-search1158+99 / −1048.009.70.6290.953292.10.0
10SBEBackward searchsearch1158+170 / −1758.278.60.2040.611830.140.0
11MIInformation theorynon-search1127+172 / −1778.086.20.7581.0000.60.0
12GainRatioInformation theorynon-search1083+106 / −1319.8810.30.6290.8120.30.0
13mRMRInformation theorynon-search1026+104 / −13210.1010.30.5170.6690.40.0
14RandomBaseline1000+131 / −22310.5516.20.0000.4740.00.0
15LaplacianScoreDistancenon-search994+125 / −19211.0715.60.6990.8250.20.0
diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_0/pareto.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_0/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..3d4ee04c5b201a68ddbb9366fcdc75b9db074179 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_0/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,13.6118,236.1137,10,True +CART,12.6442,22.3219,10,True +ElasticNet,11.6687,37.1473,10,False +Lasso,11.5261,13.4509,10,True +MI,10.8772,130.3017,10,False +F-Test,10.8449,1.0097,10,True +SFS,10.0079,14075.7296,10,False +MarkovBlanket,9.944,43.4583,10,False +SBE,8.4741,29083.1492,10,False +GainRatio,8.1389,90.0526,10,False +(R)ReliefF,7.7194,11523.6979,10,False +mRMR,7.6297,93.8878,10,False +LOCO,7.5672,8647.4113,10,False +Random,0.0,0.0621,10,True +LaplacianScore,-0.7045,4.7905,10,False diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_0/pareto.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_0/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..1cffe547a4cc26d186bc75e09f513798cc80d1b9 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_0/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 1
Hover or tap a method for its values.
10-110010110210310402468101214Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTElasticNetLassoMIF-TestSFSMarkovBlanketSBEGainRatio(R)ReliefFmRMRLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_0/website_leaderboard.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_0/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..f315f5dc15ae67707650ccf536fef750fd0df0ad --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_0/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1361.0,1229.0,1549.0,4.88,72.3,4.2,,0.733,0.85,1.0,0.0,10,30 +2,CART,Non-search,Tree,1324.0,1235.0,1446.0,5.35,68.9,5.3,,0.633,0.825,0.1,0.0,10,30 +3,SFS,Search,Forward search,1288.0,1129.0,1501.0,6.35,61.8,7.7,,0.363,0.792,245.6,30.0,10,30 +4,ElasticNet,Non-search,Regularization,1261.0,1190.0,1362.0,7.25,55.4,6.3,,0.523,0.938,0.2,0.0,10,30 +5,MarkovBlanket,Search,Backward search,1234.0,1068.0,1434.0,7.22,55.6,6.1,,0.429,0.75,0.2,0.0,10,30 +6,Lasso,Non-search,Regularization,1234.0,1170.0,1326.0,7.37,54.5,6.4,,0.541,0.922,0.1,0.0,10,30 +7,LOCO,Search,Leave-one-out,1178.0,1016.0,1324.0,7.9,50.7,10.1,,0.312,0.685,129.3,0.0,10,30 +8,F-Test,Non-search,Correlation,1163.0,1024.0,1312.0,7.73,51.9,7.0,,0.818,0.969,0.0,0.0,10,30 +9,(R)ReliefF,Non-search,Distance,1158.0,1054.0,1257.0,8.0,50.0,9.7,,0.629,0.953,292.1,0.0,10,30 +10,SBE,Search,Backward search,1158.0,983.0,1328.0,8.27,48.1,8.6,,0.204,0.611,830.1,40.0,10,30 +11,MI,Non-search,Information theory,1127.0,950.0,1299.0,8.08,49.4,6.2,,0.758,1.0,0.6,0.0,10,30 +12,GainRatio,Non-search,Information theory,1083.0,952.0,1189.0,9.88,36.6,10.3,,0.629,0.812,0.3,0.0,10,30 +13,mRMR,Non-search,Information theory,1026.0,894.0,1130.0,10.1,35.0,10.3,,0.517,0.669,0.4,0.0,10,30 +14,Random,Baseline,Baseline,1000.0,777.0,1131.0,10.55,31.8,16.2,,0.0,0.474,0.0,0.0,10,30 +15,LaplacianScore,Non-search,Distance,994.0,802.0,1119.0,11.07,28.1,15.6,,0.699,0.825,0.2,0.0,10,30 diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_1/elo_explorer.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_1/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..3f9666e37b06e6c4f6588bc669a114beb2acb463 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_1/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 2
Hover or tap a method for its values.
80010001200140016001800EloRandomLaplacianScoremRMR(R)ReliefFGainRatioSFSSBEMarkovBlanketMIF-TestLOCOElasticNetLassoRFImportanceCARTstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_1/leaderboard_table.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_1/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..1450b5e6e6064040641eca3fcfd78ffab8921193 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_1/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 2
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1CARTTreenon-search1508+140 / −915.373.30.5610.7630.10.0
🥈2RFImportanceTreenon-search1476+240 / −1685.052.70.7190.8021.00.0
🥉3LassoRegularizationnon-search1391+117 / −987.225.90.6330.8380.10.0
4ElasticNetRegularizationnon-search1380+106 / −847.055.90.6280.9000.20.0
5LOCOLeave-one-outsearch1377+187 / −1797.637.70.2320.687126.90.0
6F-TestCorrelationnon-search1340+205 / −1638.006.50.7900.9380.00.0
7MIInformation theorynon-search1318+157 / −1267.574.70.8180.9690.50.0
8MarkovBlanketBackward searchsearch1308+157 / −1517.787.50.3390.7280.10.0
9SBEBackward searchsearch1297+169 / −1707.776.10.1950.736742.240.0
10SFSForward searchsearch1260+111 / −1227.935.40.2620.711420.640.0
11GainRatioInformation theorynon-search1249+108 / −1347.976.50.6320.8050.40.0
12(R)ReliefFDistancenon-search1239+111 / −1037.977.60.6130.916291.90.0
13mRMRInformation theorynon-search1148+130 / −1689.106.90.5350.6540.40.0
14LaplacianScoreDistancenon-search1033+124 / −23811.2012.20.7200.8430.10.0
15RandomBaseline1000+122 / −21512.4016.20.0060.5150.00.0
diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_1/pareto.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_1/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..08b4b48a1e7c413a624e84f5d4cef76cb475cd4d --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_1/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,14.4166,238.5992,10,True +CART,13.9675,22.559,10,True +MI,12.5932,130.7343,10,False +SFS,11.7162,18827.2256,10,False +Lasso,11.6117,13.0903,10,True +ElasticNet,11.5981,38.0746,10,False +F-Test,10.9587,0.9982,10,True +GainRatio,10.8132,89.3045,10,False +SBE,10.682,27342.0546,10,False +mRMR,10.4885,102.2655,10,False +(R)ReliefF,9.9691,11510.395,10,False +MarkovBlanket,9.7344,41.7545,10,False +LOCO,9.3134,8693.4993,10,False +LaplacianScore,4.412,4.6874,10,False +Random,0.0,0.0033,10,True diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_1/pareto.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_1/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..564cf517c4180ef3275208030d4c8b7a5fe0bca1 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_1/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 2
Hover or tap a method for its values.
10-210-110010110210310402468101214Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMISFSLassoElasticNetF-TestGainRatioSBEmRMR(R)ReliefFMarkovBlanketLOCORandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_1/website_leaderboard.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_1/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..4390d647b3ae50cea44fbcfb60017e371efbab10 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_1/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,CART,Non-search,Tree,1508.0,1417.0,1648.0,5.37,68.8,3.3,,0.561,0.763,0.1,0.0,10,30 +2,RFImportance,Non-search,Tree,1476.0,1308.0,1716.0,5.05,71.1,2.7,,0.719,0.802,1.0,0.0,10,30 +3,Lasso,Non-search,Regularization,1391.0,1293.0,1508.0,7.22,55.6,5.9,,0.633,0.838,0.1,0.0,10,30 +4,ElasticNet,Non-search,Regularization,1380.0,1296.0,1486.0,7.05,56.8,5.9,,0.628,0.9,0.2,0.0,10,30 +5,LOCO,Search,Leave-one-out,1377.0,1198.0,1564.0,7.63,52.6,7.7,,0.232,0.687,126.9,0.0,10,30 +6,F-Test,Non-search,Correlation,1340.0,1177.0,1545.0,8.0,50.0,6.5,,0.79,0.938,0.0,0.0,10,30 +7,MI,Non-search,Information theory,1318.0,1192.0,1475.0,7.57,53.1,4.7,,0.818,0.969,0.5,0.0,10,30 +8,MarkovBlanket,Search,Backward search,1308.0,1157.0,1465.0,7.78,51.6,7.5,,0.339,0.728,0.1,0.0,10,30 +9,SBE,Search,Backward search,1297.0,1127.0,1466.0,7.77,51.7,6.1,,0.195,0.736,742.2,40.0,10,30 +10,SFS,Search,Forward search,1260.0,1138.0,1371.0,7.93,50.5,5.4,,0.262,0.711,420.6,40.0,10,30 +11,GainRatio,Non-search,Information theory,1249.0,1115.0,1357.0,7.97,50.2,6.5,,0.632,0.805,0.4,0.0,10,30 +12,(R)ReliefF,Non-search,Distance,1239.0,1136.0,1350.0,7.97,50.2,7.6,,0.613,0.916,291.9,0.0,10,30 +13,mRMR,Non-search,Information theory,1148.0,980.0,1278.0,9.1,42.1,6.9,,0.535,0.654,0.4,0.0,10,30 +14,LaplacianScore,Non-search,Distance,1033.0,795.0,1157.0,11.2,27.1,12.2,,0.72,0.843,0.1,0.0,10,30 +15,Random,Baseline,Baseline,1000.0,785.0,1122.0,12.4,18.6,16.2,,0.006,0.515,0.0,0.0,10,30 diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_2/elo_explorer.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_2/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..a5f9a275e0b5b739d757db77ca368e7e7836138c --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_2/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 3
Hover or tap a method for its values.
80010001200140016001800EloLaplacianScoreRandomF-TestGainRatioSFSmRMRMarkovBlanketMIElasticNetSBELasso(R)ReliefFLOCOCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_2/leaderboard_table.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_2/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..8a871968b7dd3619b138eff319b79172742657d7 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_2/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 3
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1587+150 / −744.852.40.6780.7051.00.0
🥈2CARTTreenon-search1554+172 / −1174.982.20.5170.6670.10.0
🥉3LOCOLeave-one-outsearch1407+195 / −1546.274.50.1600.668126.70.0
4(R)ReliefFDistancenon-search1402+113 / −1056.534.40.7040.842291.10.0
5LassoRegularizationnon-search1356+198 / −1946.725.40.4910.7840.10.0
6SBEBackward searchsearch1312+169 / −1557.836.80.1510.608548.840.0
7ElasticNetRegularizationnon-search1301+167 / −1677.675.80.5600.8360.20.0
8MIInformation theorynon-search1290+152 / −1508.474.90.9170.9110.50.0
9MarkovBlanketBackward searchsearch1279+134 / −1329.037.80.3850.6820.10.0
10mRMRInformation theorynon-search1243+142 / −1459.357.20.5420.5930.50.0
11SFSForward searchsearch1224+135 / −1248.236.20.2600.645795.740.0
12GainRatioInformation theorynon-search1221+119 / −1249.128.10.5640.7410.40.0
13F-TestCorrelationnon-search1172+124 / −1709.186.10.7640.8730.00.0
14RandomBaseline1000+87 / −18110.5010.9-0.0030.4900.00.0
15LaplacianScoreDistancenon-search991+125 / −25011.2712.60.7650.8490.10.0
diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_2/pareto.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_2/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..e01c972cb9509660ab052d7653e40cb23b936dd9 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_2/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +CART,9.475,22.1766,10,True +RFImportance,9.2807,234.7472,10,False +(R)ReliefF,7.4819,11516.6297,10,False +LOCO,7.1338,8306.1025,10,False +MI,6.7913,131.2443,10,False +Lasso,6.4369,13.4973,10,True +ElasticNet,6.0538,36.5583,10,False +F-Test,6.0515,0.9973,10,True +SFS,5.1055,24787.8,10,False +SBE,4.8166,23603.162,10,False +mRMR,4.3187,119.1243,10,False +MarkovBlanket,3.9453,41.7802,10,False +GainRatio,3.5168,90.7532,10,False +Random,0.0,0.0066,10,True +LaplacianScore,-2.2408,5.2201,10,False diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_2/pareto.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_2/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..c91b8d48b4b00633d503047bb9127cbc302b2415 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_2/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 3
Hover or tap a method for its values.
10-210-110010110210310402468Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalCARTRFImportance(R)ReliefFLOCOMILassoElasticNetF-TestSFSSBEmRMRMarkovBlanketGainRatioRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_2/website_leaderboard.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_2/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..a0eeb5c07e988eb9192bba682fd9bad425524714 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_2/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1587.0,1513.0,1737.0,4.85,72.5,2.4,,0.678,0.705,1.0,0.0,10,30 +2,CART,Non-search,Tree,1554.0,1437.0,1726.0,4.98,71.6,2.2,,0.517,0.667,0.1,0.0,10,30 +3,LOCO,Search,Leave-one-out,1407.0,1253.0,1602.0,6.27,62.4,4.5,,0.16,0.668,126.7,0.0,10,30 +4,(R)ReliefF,Non-search,Distance,1402.0,1297.0,1515.0,6.53,60.5,4.4,,0.704,0.842,291.1,0.0,10,30 +5,Lasso,Non-search,Regularization,1356.0,1162.0,1554.0,6.72,59.2,5.4,,0.491,0.784,0.1,0.0,10,30 +6,SBE,Search,Backward search,1312.0,1157.0,1481.0,7.83,51.2,6.8,,0.151,0.608,548.8,40.0,10,30 +7,ElasticNet,Non-search,Regularization,1301.0,1134.0,1468.0,7.67,52.4,5.8,,0.56,0.836,0.2,0.0,10,30 +8,MI,Non-search,Information theory,1290.0,1140.0,1442.0,8.47,46.7,4.9,,0.917,0.911,0.5,0.0,10,30 +9,MarkovBlanket,Search,Backward search,1279.0,1147.0,1413.0,9.03,42.6,7.8,,0.385,0.682,0.1,0.0,10,30 +10,mRMR,Non-search,Information theory,1243.0,1098.0,1385.0,9.35,40.4,7.2,,0.542,0.593,0.5,0.0,10,30 +11,SFS,Search,Forward search,1224.0,1100.0,1359.0,8.23,48.3,6.2,,0.26,0.645,795.7,40.0,10,30 +12,GainRatio,Non-search,Information theory,1221.0,1097.0,1340.0,9.12,42.0,8.1,,0.564,0.741,0.4,0.0,10,30 +13,F-Test,Non-search,Correlation,1172.0,1002.0,1296.0,9.18,41.6,6.1,,0.764,0.873,0.0,0.0,10,30 +14,Random,Baseline,Baseline,1000.0,819.0,1087.0,10.5,32.1,10.9,,-0.003,0.49,0.0,0.0,10,30 +15,LaplacianScore,Non-search,Distance,991.0,741.0,1116.0,11.27,26.7,12.6,,0.765,0.849,0.1,0.0,10,30 diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_3/elo_explorer.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_3/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..9df7e0e999600374c89c1d184cfcc592606f9c55 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_3/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 4
Hover or tap a method for its values.
60080010001200140016001800EloLaplacianScoreRandomSBEElasticNetMarkovBlanketmRMRGainRatioMISFSF-TestLassoLOCO(R)ReliefFCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_3/leaderboard_table.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_3/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..9822d923882af31f373ff9b8f1e247548d44cd65 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_3/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 4
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1471+169 / −1054.752.00.7340.6071.00.0
🥈2CARTTreenon-search1405+147 / −805.522.10.4890.5460.10.0
🥉3(R)ReliefFDistancenon-search1386+138 / −1025.402.80.7310.741282.40.0
4LOCOLeave-one-outsearch1340+155 / −1266.174.00.0920.621123.910.0
5LassoRegularizationnon-search1306+212 / −1757.284.60.4230.7090.10.0
6F-TestCorrelationnon-search1223+140 / −1197.804.00.7130.7780.00.0
7SFSForward searchsearch1218+155 / −1587.505.10.2180.598931.240.0
8MIInformation theorynon-search1212+124 / −1238.304.40.7960.7740.60.0
9GainRatioInformation theorynon-search1204+123 / −1388.726.10.5590.6350.30.0
10mRMRInformation theorynon-search1199+90 / −1168.925.80.5540.5330.50.0
11MarkovBlanketBackward searchsearch1196+175 / −1629.376.60.4150.6340.10.0
12ElasticNetRegularizationnon-search1188+162 / −1708.355.00.4780.7250.20.0
13SBEBackward searchsearch1180+145 / −1528.836.30.0890.588492.840.0
14RandomBaseline1000+126 / −23511.009.20.0030.5200.00.0
15LaplacianScoreDistancenon-search946+169 / −35412.1012.50.7590.7270.10.0
diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_3/pareto.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_3/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..8076823ee812a2c150a5e0ea22be678cb5bae502 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_3/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,7.7333,239.7182,10,True +CART,7.725,22.3707,10,True +(R)ReliefF,7.024,11505.1892,10,False +F-Test,5.8107,0.9837,10,True +MI,5.6356,132.1318,10,False +LOCO,5.5592,8100.1443,10,False +Lasso,5.0298,12.6567,10,False +ElasticNet,4.7709,36.4189,10,False +SFS,4.2023,26839.4511,10,False +mRMR,3.993,152.909,10,False +GainRatio,3.5949,89.8989,10,False +SBE,3.418,20879.9148,10,False +MarkovBlanket,3.2642,40.8445,10,False +Random,0.0,0.0035,10,True +LaplacianScore,-4.6637,4.9009,10,False diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_3/pareto.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_3/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..df6c89d889105da21fc0aac5053e9833cc0fe276 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_3/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 4
Hover or tap a method for its values.
10-210-1100101102103104012345678Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCART(R)ReliefFF-TestMILOCOLassoElasticNetSFSmRMRGainRatioSBEMarkovBlanketRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_3/website_leaderboard.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_3/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..b640b0f9e00e0f906682378603e74e468713414c --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_3/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1471.0,1366.0,1640.0,4.75,73.2,2.0,,0.734,0.607,1.0,0.0,10,30 +2,CART,Non-search,Tree,1405.0,1325.0,1552.0,5.52,67.7,2.1,,0.489,0.546,0.1,0.0,10,30 +3,(R)ReliefF,Non-search,Distance,1386.0,1284.0,1524.0,5.4,68.6,2.8,,0.731,0.741,282.4,0.0,10,30 +4,LOCO,Search,Leave-one-out,1340.0,1214.0,1495.0,6.17,63.1,4.0,,0.092,0.621,123.9,10.0,10,30 +5,Lasso,Non-search,Regularization,1306.0,1131.0,1518.0,7.28,55.1,4.6,,0.423,0.709,0.1,0.0,10,30 +6,F-Test,Non-search,Correlation,1223.0,1104.0,1363.0,7.8,51.4,4.0,,0.713,0.778,0.0,0.0,10,30 +7,SFS,Search,Forward search,1218.0,1060.0,1373.0,7.5,53.6,5.1,,0.218,0.598,931.2,40.0,10,30 +8,MI,Non-search,Information theory,1212.0,1089.0,1336.0,8.3,47.9,4.4,,0.796,0.774,0.6,0.0,10,30 +9,GainRatio,Non-search,Information theory,1204.0,1066.0,1327.0,8.72,44.9,6.1,,0.559,0.635,0.3,0.0,10,30 +10,mRMR,Non-search,Information theory,1199.0,1083.0,1289.0,8.92,43.4,5.8,,0.554,0.533,0.5,0.0,10,30 +11,MarkovBlanket,Search,Backward search,1196.0,1034.0,1371.0,9.37,40.2,6.6,,0.415,0.634,0.1,0.0,10,30 +12,ElasticNet,Non-search,Regularization,1188.0,1018.0,1350.0,8.35,47.5,5.0,,0.478,0.725,0.2,0.0,10,30 +13,SBE,Search,Backward search,1180.0,1028.0,1325.0,8.83,44.0,6.3,,0.089,0.588,492.8,40.0,10,30 +14,Random,Baseline,Baseline,1000.0,765.0,1126.0,11.0,28.6,9.2,,0.003,0.52,0.0,0.0,10,30 +15,LaplacianScore,Non-search,Distance,946.0,592.0,1115.0,12.1,20.7,12.5,,0.759,0.727,0.1,0.0,10,30 diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_4/elo_explorer.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_4/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..4922e1c4e88fce8b643eaa921ffc65dca39fb949 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_4/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 5
Hover or tap a method for its values.
8001000120014001600EloRandomLaplacianScoreSBEMarkovBlanketMImRMRLOCOF-TestGainRatioSFSCARTLassoElasticNetRFImportance(R)ReliefFstableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_4/leaderboard_table.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_4/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..186b8a74524a926958c2e4009d31894f8e45b051 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_4/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 5
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1(R)ReliefFDistancenon-search1364+178 / −1035.801.60.6650.634281.30.0
🥈2RFImportanceTreenon-search1308+163 / −1376.201.60.7060.4811.00.0
🥉3ElasticNetRegularizationnon-search1290+153 / −1186.272.10.4570.6170.20.0
4LassoRegularizationnon-search1284+153 / −1256.232.20.4150.6030.10.0
5CARTTreenon-search1277+146 / −1026.971.30.5060.4630.10.0
6SFSForward searchsearch1226+171 / −1738.103.50.1200.533917.240.0
7F-TestCorrelationnon-search1206+150 / −1467.672.40.5510.6880.00.0
8GainRatioInformation theorynon-search1206+124 / −1317.853.20.5040.5530.40.0
9LOCOLeave-one-outsearch1191+143 / −1408.272.40.0520.557124.920.0
10mRMRInformation theorynon-search1181+108 / −1358.683.10.5750.5170.50.0
11MIInformation theorynon-search1166+124 / −1228.402.50.7590.6480.60.0
12MarkovBlanketBackward searchsearch1146+144 / −1598.703.00.4350.5610.10.0
13SBEBackward searchsearch1131+135 / −1418.933.50.0610.541246.440.0
14LaplacianScoreDistancenon-search1025+171 / −26611.109.10.7550.6340.10.0
15RandomBaseline1000+82 / −13810.836.0-0.0010.4930.00.0
diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_4/pareto.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_4/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..1c1305bbd1743d65c001f1711255ee9f5d8860d9 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_4/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +CART,4.6097,22.1399,10,True +(R)ReliefF,4.501,11499.3544,10,False +RFImportance,4.3792,232.8453,10,False +ElasticNet,3.9536,37.258,10,False +Lasso,3.9156,12.3901,10,True +LOCO,3.7384,8323.1609,10,False +MI,3.6847,129.1837,10,False +F-Test,3.6346,0.9629,10,True +MarkovBlanket,3.0933,39.2379,10,False +mRMR,2.9571,237.9148,10,False +GainRatio,2.9206,89.6556,10,False +SBE,2.628,14873.4492,10,False +SFS,2.1356,27960.2996,10,False +Random,0.0,0.0036,10,True +LaplacianScore,-4.1187,4.9179,10,False diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_4/pareto.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_4/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..1e6638119ee561d18c71ae6ed7b3da07071779a0 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_4/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, TabICLv2, budget 5
Hover or tap a method for its values.
10-210-110010110210310401234Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalCART(R)ReliefFRFImportanceElasticNetLassoLOCOMIF-TestMarkovBlanketmRMRGainRatioSBESFSRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_4/website_leaderboard.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_4/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..6830b410672a510ce16213aff01d277e924ef2f3 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_4/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,(R)ReliefF,Non-search,Distance,1364.0,1261.0,1542.0,5.8,65.7,1.6,,0.665,0.634,281.3,0.0,10,30 +2,RFImportance,Non-search,Tree,1308.0,1171.0,1471.0,6.2,62.9,1.6,,0.706,0.481,1.0,0.0,10,30 +3,ElasticNet,Non-search,Regularization,1290.0,1172.0,1443.0,6.27,62.4,2.1,,0.457,0.617,0.2,0.0,10,30 +4,Lasso,Non-search,Regularization,1284.0,1159.0,1437.0,6.23,62.6,2.2,,0.415,0.603,0.1,0.0,10,30 +5,CART,Non-search,Tree,1277.0,1175.0,1423.0,6.97,57.4,1.3,,0.506,0.463,0.1,0.0,10,30 +6,SFS,Search,Forward search,1226.0,1053.0,1397.0,8.1,49.3,3.5,,0.12,0.533,917.2,40.0,10,30 +7,F-Test,Non-search,Correlation,1206.0,1060.0,1356.0,7.67,52.4,2.4,,0.551,0.688,0.0,0.0,10,30 +8,GainRatio,Non-search,Information theory,1206.0,1075.0,1330.0,7.85,51.1,3.2,,0.504,0.553,0.4,0.0,10,30 +9,LOCO,Search,Leave-one-out,1191.0,1051.0,1334.0,8.27,48.1,2.4,,0.052,0.557,124.9,20.0,10,30 +10,mRMR,Non-search,Information theory,1181.0,1046.0,1289.0,8.68,45.1,3.1,,0.575,0.517,0.5,0.0,10,30 +11,MI,Non-search,Information theory,1166.0,1044.0,1290.0,8.4,47.1,2.5,,0.759,0.648,0.6,0.0,10,30 +12,MarkovBlanket,Search,Backward search,1146.0,987.0,1290.0,8.7,45.0,3.0,,0.435,0.561,0.1,0.0,10,30 +13,SBE,Search,Backward search,1131.0,990.0,1266.0,8.93,43.3,3.5,,0.061,0.541,246.4,40.0,10,30 +14,LaplacianScore,Non-search,Distance,1025.0,759.0,1196.0,11.1,27.9,9.1,,0.755,0.634,0.1,0.0,10,30 +15,Random,Baseline,Baseline,1000.0,862.0,1082.0,10.83,29.8,6.0,,-0.001,0.493,0.0,0.0,10,30 diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_all/elo_explorer.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_all/elo_explorer.html new file mode 100644 index 0000000000000000000000000000000000000000..072dca0858bb7194ae7422ad7113525bd447102e --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_all/elo_explorer.html @@ -0,0 +1,154 @@ + +SelectArena leaderboard: regression, TabICLv2, all budgets
Hover or tap a method for its values.
8009001000110012001300140015001600EloLaplacianScoreRandommRMRGainRatioSBEF-TestMIMarkovBlanketSFSElasticNetLOCO(R)ReliefFLassoCARTRFImportancestableunstable0.00.51.0validinvalid0.00.51.0no result diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_all/leaderboard_table.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_all/leaderboard_table.html new file mode 100644 index 0000000000000000000000000000000000000000..2e199902f702566c9031227a37f409866f6d45f0 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_all/leaderboard_table.html @@ -0,0 +1,112 @@ + +SelectArena leaderboard: regression, TabICLv2, all budgets
Click a column to sort. Hover a header for its definition. Stability and validity are shaded as in the Elo figure: darker is better.
#MethodTypeElo ↑Avg. rank ↓Improvability ↓Stability ↑Validity ↑Time (s) ↓Fallback (%)
🥇1RFImportanceTreenon-search1422+119 / −965.152.60.7140.6891.00.0
🥈2CARTTreenon-search1396+105 / −655.642.80.5410.6530.10.0
🥉3LassoRegularizationnon-search1304+110 / −1016.964.90.5000.7710.10.0
4(R)ReliefFDistancenon-search1297+77 / −656.745.20.6690.817291.90.0
5LOCOLeave-one-outsearch1287+92 / −987.255.70.1700.643125.66.0
6ElasticNetRegularizationnon-search1275+100 / −937.325.00.5290.8030.20.0
7SFSForward searchsearch1236+86 / −877.625.60.2550.656487.538.0
8MarkovBlanketBackward searchsearch1224+74 / −808.426.20.4010.6710.10.0
9F-TestCorrelationnon-search1214+111 / −1048.085.20.7270.8490.00.0
10MIInformation theorynon-search1214+96 / −958.164.60.8100.8600.60.0
11SBEBackward searchsearch1207+118 / −1178.336.30.1400.617613.040.0
12GainRatioInformation theorynon-search1186+72 / −958.716.80.5780.7090.40.0
13mRMRInformation theorynon-search1155+90 / −1139.236.70.5450.5930.50.0
14RandomBaseline1000+72 / −11711.0611.70.0010.4980.00.0
15LaplacianScoreDistancenon-search999+127 / −18111.3512.40.7400.7760.10.0
diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_all/pareto.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_all/pareto.csv new file mode 100644 index 0000000000000000000000000000000000000000..ca611bdf409f6e99926c36b21ccec043bc1720cc --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_all/pareto.csv @@ -0,0 +1,16 @@ +method,improvement,time_per_1k,n_datasets,pareto_optimal +RFImportance,10.6915,236.4047,10,True +CART,10.4596,22.3136,10,True +MI,9.019,130.7191,10,False +Lasso,8.4795,13.0171,10,True +ElasticNet,8.3946,37.0914,10,False +F-Test,7.9497,0.9903,10,True +(R)ReliefF,7.4877,11511.0532,10,False +SFS,7.3697,22498.1012,10,False +MarkovBlanket,6.9946,41.4151,10,False +LOCO,6.9491,8414.0637,10,False +SBE,6.8632,23156.346,10,False +GainRatio,6.5338,89.933,10,False +mRMR,6.3499,141.2203,10,False +Random,0.0,0.0158,10,True +LaplacianScore,-0.3587,4.9034,10,False diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_all/pareto.html b/data/subsets/tasks_regression/models_tabiclv2/budgets_all/pareto.html new file mode 100644 index 0000000000000000000000000000000000000000..27a8954ee55d0bdaa92e507910086652592c73c1 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_all/pareto.html @@ -0,0 +1,161 @@ + +SelectArena leaderboard: regression, TabICLv2, all budgets
Hover or tap a method for its values.
10-210-11001011021031040246810Feature Selection Time (s) per 1k FeaturesImprovement over Random (%)optimalRFImportanceCARTMILassoElasticNetF-Test(R)ReliefFSFSMarkovBlanketLOCOSBEGainRatiomRMRRandomDominated MethodsPareto Front diff --git a/data/subsets/tasks_regression/models_tabiclv2/budgets_all/website_leaderboard.csv b/data/subsets/tasks_regression/models_tabiclv2/budgets_all/website_leaderboard.csv new file mode 100644 index 0000000000000000000000000000000000000000..349e30cd4c0e862a266eb7274699e3389832c286 --- /dev/null +++ b/data/subsets/tasks_regression/models_tabiclv2/budgets_all/website_leaderboard.csv @@ -0,0 +1,16 @@ +Rank,Method,Category,Subgroup,Elo,Elo CI low,Elo CI high,Avg. rank,Win rate (%),Improvability (%),Error vs. all features,Stability,Validity,Selection time (s),Fallback rate (%),#Datasets,#Comparisons +1,RFImportance,Non-search,Tree,1422.0,1326.0,1541.0,5.15,70.4,2.6,,0.714,0.689,1.0,0.0,10,150 +2,CART,Non-search,Tree,1396.0,1331.0,1501.0,5.64,66.9,2.8,,0.541,0.653,0.1,0.0,10,150 +3,Lasso,Non-search,Regularization,1304.0,1203.0,1414.0,6.96,57.4,4.9,,0.5,0.771,0.1,0.0,10,150 +4,(R)ReliefF,Non-search,Distance,1297.0,1232.0,1374.0,6.74,59.0,5.2,,0.669,0.817,291.9,0.0,10,150 +5,LOCO,Search,Leave-one-out,1287.0,1189.0,1379.0,7.25,55.4,5.7,,0.17,0.643,125.6,6.0,10,150 +6,ElasticNet,Non-search,Regularization,1275.0,1182.0,1375.0,7.32,54.9,5.0,,0.529,0.803,0.2,0.0,10,150 +7,SFS,Search,Forward search,1236.0,1149.0,1322.0,7.62,52.7,5.6,,0.255,0.656,487.5,38.0,10,150 +8,MarkovBlanket,Search,Backward search,1224.0,1144.0,1298.0,8.42,47.0,6.2,,0.401,0.671,0.1,0.0,10,150 +9,F-Test,Non-search,Correlation,1214.0,1110.0,1325.0,8.08,49.4,5.2,,0.727,0.849,0.0,0.0,10,150 +10,MI,Non-search,Information theory,1214.0,1119.0,1310.0,8.16,48.8,4.6,,0.81,0.86,0.6,0.0,10,150 +11,SBE,Search,Backward search,1207.0,1090.0,1325.0,8.33,47.7,6.3,,0.14,0.617,613.0,40.0,10,150 +12,GainRatio,Non-search,Information theory,1186.0,1091.0,1258.0,8.71,45.0,6.8,,0.578,0.709,0.4,0.0,10,150 +13,mRMR,Non-search,Information theory,1155.0,1042.0,1245.0,9.23,41.2,6.7,,0.545,0.593,0.5,0.0,10,150 +14,Random,Baseline,Baseline,1000.0,883.0,1072.0,11.06,28.2,11.7,,0.001,0.498,0.0,0.0,10,150 +15,LaplacianScore,Non-search,Distance,999.0,818.0,1126.0,11.35,26.1,12.4,,0.74,0.776,0.1,0.0,10,150